2024-04-22T21:00:12.162 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-22T21:00:12.162 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2024-04-22_20:08:13-orch-main-distro-default-smithi/7668628 --verbose --description orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/connectivity} --name teuthology-2024-04-22_20:08:13-orch-main-distro-default-smithi --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-04-22_20:08:13-orch-main-distro-default-smithi/7668628/orig.config.yaml 2024-04-22T21:00:12.208 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-22T21:00:12.292 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-22_20:08:13-orch-main-distro-default-smithi/7668628 branch: main description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/connectivity} email: ceph-qa@ceph.com first_in_suite: false job_id: '7668628' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-04-22_20:08:13-orch-main-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: 56e81df3aeb98b717efe2ab0537a8c60249f95f7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 56e81df3aeb98b717efe2ab0537a8c60249f95f7 workunit: branch: main sha1: 8254cb5a247caa8e3d35e534a1fefe2362a0a4b8 owner: scheduled_teuthology@teuthology priority: 950 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 6458 sha1: 56e81df3aeb98b717efe2ab0537a8c60249f95f7 sleep_before_teardown: 0 subset: 39/64 suite: orch suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_8254cb5a247caa8e3d35e534a1fefe2362a0a4b8/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 8254cb5a247caa8e3d35e534a1fefe2362a0a4b8 targets: smithi130.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCZqXFK3BX5T+UkFp0iVOoBqAHNV6hSUhvpaASbi9orSRUA16kXCdgPp6dN43rp12AEeAfj/Dy9ArZ9QWGx7ss8= smithi150.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJalP1qhv6op7hcldCkiTf9xivMy4N4V8YnZVFrN437D71AZDtaE9jr3O5Q46Me8x0CpEd+cS3Fovw506iw4cjw= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest false --force - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-22_20:08:13 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-22T21:00:12.292 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_8254cb5a247caa8e3d35e534a1fefe2362a0a4b8/qa; will attempt to use it 2024-04-22T21:00:12.293 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_8254cb5a247caa8e3d35e534a1fefe2362a0a4b8/qa/tasks 2024-04-22T21:00:12.293 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-22T21:00:12.295 INFO:teuthology.task.internal:Checking packages... 2024-04-22T21:00:12.320 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '56e81df3aeb98b717efe2ab0537a8c60249f95f7' 2024-04-22T21:00:12.320 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-22T21:00:12.320 INFO:teuthology.packaging:ref: None 2024-04-22T21:00:12.320 INFO:teuthology.packaging:tag: None 2024-04-22T21:00:12.320 INFO:teuthology.packaging:branch: main 2024-04-22T21:00:12.320 INFO:teuthology.packaging:sha1: 56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:00:12.321 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-04-22T21:00:12.645 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3198.g56e81df3 2024-04-22T21:00:12.646 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-22T21:00:12.652 INFO:teuthology.task.internal:no buildpackages task found 2024-04-22T21:00:12.652 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-22T21:00:12.671 INFO:teuthology.task.internal:Saving configuration 2024-04-22T21:00:12.690 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-22T21:00:12.695 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-22T21:00:12.720 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi130.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-22_20:08:13-orch-main-distro-default-smithi/7668628', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 20:54:54.956186', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCZqXFK3BX5T+UkFp0iVOoBqAHNV6hSUhvpaASbi9orSRUA16kXCdgPp6dN43rp12AEeAfj/Dy9ArZ9QWGx7ss8='} 2024-04-22T21:00:12.741 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi150.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-22_20:08:13-orch-main-distro-default-smithi/7668628', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 20:54:54.957824', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJalP1qhv6op7hcldCkiTf9xivMy4N4V8YnZVFrN437D71AZDtaE9jr3O5Q46Me8x0CpEd+cS3Fovw506iw4cjw='} 2024-04-22T21:00:12.741 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-22T21:00:12.747 INFO:teuthology.task.internal:roles: ubuntu@smithi130.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-22T21:00:12.747 INFO:teuthology.task.internal:roles: ubuntu@smithi150.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-22T21:00:12.747 INFO:teuthology.run_tasks:Running task console_log... 2024-04-22T21:00:12.818 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f67dddac0d0>, signals=[15]) 2024-04-22T21:00:12.818 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-22T21:00:12.824 INFO:teuthology.task.internal:Opening connections... 2024-04-22T21:00:12.824 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi130.front.sepia.ceph.com 2024-04-22T21:00:12.826 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T21:00:12.904 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi150.front.sepia.ceph.com 2024-04-22T21:00:12.905 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T21:00:12.980 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-22T21:00:12.988 DEBUG:teuthology.orchestra.run.smithi130:> uname -m 2024-04-22T21:00:13.006 INFO:teuthology.orchestra.run.smithi130.stdout:x86_64 2024-04-22T21:00:13.007 DEBUG:teuthology.orchestra.run.smithi130:> cat /etc/os-release 2024-04-22T21:00:13.064 INFO:teuthology.orchestra.run.smithi130.stdout:NAME="CentOS Stream" 2024-04-22T21:00:13.064 INFO:teuthology.orchestra.run.smithi130.stdout:VERSION="9" 2024-04-22T21:00:13.064 INFO:teuthology.orchestra.run.smithi130.stdout:ID="centos" 2024-04-22T21:00:13.064 INFO:teuthology.orchestra.run.smithi130.stdout:ID_LIKE="rhel fedora" 2024-04-22T21:00:13.064 INFO:teuthology.orchestra.run.smithi130.stdout:VERSION_ID="9" 2024-04-22T21:00:13.064 INFO:teuthology.orchestra.run.smithi130.stdout:PLATFORM_ID="platform:el9" 2024-04-22T21:00:13.064 INFO:teuthology.orchestra.run.smithi130.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-22T21:00:13.065 INFO:teuthology.orchestra.run.smithi130.stdout:ANSI_COLOR="0;31" 2024-04-22T21:00:13.065 INFO:teuthology.orchestra.run.smithi130.stdout:LOGO="fedora-logo-icon" 2024-04-22T21:00:13.065 INFO:teuthology.orchestra.run.smithi130.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-22T21:00:13.065 INFO:teuthology.orchestra.run.smithi130.stdout:HOME_URL="https://centos.org/" 2024-04-22T21:00:13.065 INFO:teuthology.orchestra.run.smithi130.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-22T21:00:13.065 INFO:teuthology.orchestra.run.smithi130.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-22T21:00:13.065 INFO:teuthology.orchestra.run.smithi130.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-22T21:00:13.066 INFO:teuthology.lock.ops:Updating smithi130.front.sepia.ceph.com on lock server 2024-04-22T21:00:13.088 DEBUG:teuthology.orchestra.run.smithi150:> uname -m 2024-04-22T21:00:13.107 INFO:teuthology.orchestra.run.smithi150.stdout:x86_64 2024-04-22T21:00:13.107 DEBUG:teuthology.orchestra.run.smithi150:> cat /etc/os-release 2024-04-22T21:00:13.165 INFO:teuthology.orchestra.run.smithi150.stdout:NAME="CentOS Stream" 2024-04-22T21:00:13.165 INFO:teuthology.orchestra.run.smithi150.stdout:VERSION="9" 2024-04-22T21:00:13.165 INFO:teuthology.orchestra.run.smithi150.stdout:ID="centos" 2024-04-22T21:00:13.165 INFO:teuthology.orchestra.run.smithi150.stdout:ID_LIKE="rhel fedora" 2024-04-22T21:00:13.165 INFO:teuthology.orchestra.run.smithi150.stdout:VERSION_ID="9" 2024-04-22T21:00:13.165 INFO:teuthology.orchestra.run.smithi150.stdout:PLATFORM_ID="platform:el9" 2024-04-22T21:00:13.166 INFO:teuthology.orchestra.run.smithi150.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-22T21:00:13.166 INFO:teuthology.orchestra.run.smithi150.stdout:ANSI_COLOR="0;31" 2024-04-22T21:00:13.166 INFO:teuthology.orchestra.run.smithi150.stdout:LOGO="fedora-logo-icon" 2024-04-22T21:00:13.166 INFO:teuthology.orchestra.run.smithi150.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-22T21:00:13.166 INFO:teuthology.orchestra.run.smithi150.stdout:HOME_URL="https://centos.org/" 2024-04-22T21:00:13.166 INFO:teuthology.orchestra.run.smithi150.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-22T21:00:13.166 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-22T21:00:13.166 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-22T21:00:13.166 INFO:teuthology.lock.ops:Updating smithi150.front.sepia.ceph.com on lock server 2024-04-22T21:00:13.188 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-22T21:00:13.197 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-22T21:00:13.202 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-22T21:00:13.202 DEBUG:teuthology.orchestra.run.smithi130:> test '!' -e /home/ubuntu/cephtest 2024-04-22T21:00:13.205 DEBUG:teuthology.orchestra.run.smithi150:> test '!' -e /home/ubuntu/cephtest 2024-04-22T21:00:13.221 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-22T21:00:13.227 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-22T21:00:13.227 DEBUG:teuthology.orchestra.run.smithi130:> test -z $(ls -A /var/lib/ceph) 2024-04-22T21:00:13.262 DEBUG:teuthology.orchestra.run.smithi150:> test -z $(ls -A /var/lib/ceph) 2024-04-22T21:00:13.351 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-22T21:00:13.407 INFO:teuthology.run_tasks:Running task kernel... 2024-04-22T21:00:13.421 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-22T21:00:13.421 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-22T21:00:13.421 DEBUG:teuthology.orchestra.run.smithi130:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-22T21:00:13.422 DEBUG:teuthology.orchestra.run.smithi150:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-22T21:00:13.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T21:00:13.438 DEBUG:teuthology.orchestra.run.smithi150:> uname -r 2024-04-22T21:00:13.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T21:00:13.439 DEBUG:teuthology.orchestra.run.smithi130:> uname -r 2024-04-22T21:00:13.494 INFO:teuthology.orchestra.run.smithi150.stdout:5.14.0-437.el9.x86_64 2024-04-22T21:00:13.495 INFO:teuthology.task.kernel:Running kernel on smithi150: 5.14.0-437.el9.x86_64 2024-04-22T21:00:13.495 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum install -y kernel 2024-04-22T21:00:13.496 INFO:teuthology.orchestra.run.smithi130.stdout:5.14.0-437.el9.x86_64 2024-04-22T21:00:13.496 INFO:teuthology.task.kernel:Running kernel on smithi130: 5.14.0-437.el9.x86_64 2024-04-22T21:00:13.496 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum install -y kernel 2024-04-22T21:00:15.860 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - BaseOS 9.8 MB/s | 8.0 MB 00:00 2024-04-22T21:00:16.160 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - BaseOS 8.5 MB/s | 8.0 MB 00:00 2024-04-22T21:00:17.992 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - AppStream 25 MB/s | 19 MB 00:00 2024-04-22T21:00:20.559 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - AppStream 5.2 MB/s | 19 MB 00:03 2024-04-22T21:00:23.626 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - CRB 3.0 MB/s | 6.2 MB 00:02 2024-04-22T21:00:25.026 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - Extras packages 66 kB/s | 16 kB 00:00 2024-04-22T21:00:25.034 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - CRB 7.4 MB/s | 6.2 MB 00:00 2024-04-22T21:00:26.427 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - Extras packages 71 kB/s | 16 kB 00:00 2024-04-22T21:00:26.947 INFO:teuthology.orchestra.run.smithi150.stdout:Extra Packages for Enterprise Linux 12 MB/s | 21 MB 00:01 2024-04-22T21:00:27.814 INFO:teuthology.orchestra.run.smithi130.stdout:Extra Packages for Enterprise Linux 16 MB/s | 21 MB 00:01 2024-04-22T21:00:32.187 INFO:teuthology.orchestra.run.smithi150.stdout:lab-extras 6.9 kB/s | 1.7 kB 00:00 2024-04-22T21:00:32.903 INFO:teuthology.orchestra.run.smithi130.stdout:lab-extras 11 kB/s | 1.7 kB 00:00 2024-04-22T21:00:34.009 INFO:teuthology.orchestra.run.smithi150.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-22T21:00:34.009 INFO:teuthology.orchestra.run.smithi150.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-22T21:00:34.094 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-22T21:00:34.095 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-04-22T21:00:34.095 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-22T21:00:34.211 DEBUG:teuthology.orchestra.run.smithi150:> echo no | sudo yum reinstall kernel || true 2024-04-22T21:00:34.714 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:02 ago on Mon 22 Apr 2024 09:00:32 PM UTC. 2024-04-22T21:00:34.759 INFO:teuthology.orchestra.run.smithi130.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-22T21:00:34.759 INFO:teuthology.orchestra.run.smithi130.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-22T21:00:34.839 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-04-22T21:00:34.840 INFO:teuthology.orchestra.run.smithi130.stdout:Nothing to do. 2024-04-22T21:00:34.840 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-04-22T21:00:34.840 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-22T21:00:34.846 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-22T21:00:34.846 INFO:teuthology.orchestra.run.smithi150.stdout: Package Architecture Version Repository Size 2024-04-22T21:00:34.846 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-22T21:00:34.846 INFO:teuthology.orchestra.run.smithi150.stdout:Reinstalling: 2024-04-22T21:00:34.846 INFO:teuthology.orchestra.run.smithi150.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T21:00:34.846 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-22T21:00:34.846 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-22T21:00:34.846 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-22T21:00:34.847 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-22T21:00:34.847 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 255 k 2024-04-22T21:00:34.847 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 0 2024-04-22T21:00:34.847 INFO:teuthology.orchestra.run.smithi150.stderr:Operation aborted. 2024-04-22T21:00:34.908 INFO:teuthology.orchestra.run.smithi150.stdout:Is this ok [y/N]: 2024-04-22T21:00:34.908 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum reinstall -y kernel || true 2024-04-22T21:00:34.954 DEBUG:teuthology.orchestra.run.smithi130:> echo no | sudo yum reinstall kernel || true 2024-04-22T21:00:35.409 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:03 ago on Mon 22 Apr 2024 09:00:32 PM UTC. 2024-04-22T21:00:35.457 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:00:03 ago on Mon 22 Apr 2024 09:00:32 PM UTC. 2024-04-22T21:00:35.524 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-22T21:00:35.526 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-22T21:00:35.526 INFO:teuthology.orchestra.run.smithi150.stdout: Package Architecture Version Repository Size 2024-04-22T21:00:35.526 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-22T21:00:35.526 INFO:teuthology.orchestra.run.smithi150.stdout:Reinstalling: 2024-04-22T21:00:35.526 INFO:teuthology.orchestra.run.smithi150.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T21:00:35.526 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-22T21:00:35.526 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-22T21:00:35.527 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-22T21:00:35.527 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-22T21:00:35.527 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 255 k 2024-04-22T21:00:35.527 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 0 2024-04-22T21:00:35.527 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-22T21:00:35.577 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-04-22T21:00:35.587 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-04-22T21:00:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: Package Architecture Version Repository Size 2024-04-22T21:00:35.588 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-04-22T21:00:35.588 INFO:teuthology.orchestra.run.smithi130.stdout:Reinstalling: 2024-04-22T21:00:35.588 INFO:teuthology.orchestra.run.smithi130.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T21:00:35.588 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:00:35.588 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-04-22T21:00:35.588 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-04-22T21:00:35.588 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:00:35.588 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 255 k 2024-04-22T21:00:35.589 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 0 2024-04-22T21:00:35.589 INFO:teuthology.orchestra.run.smithi130.stderr:Operation aborted. 2024-04-22T21:00:35.649 INFO:teuthology.orchestra.run.smithi130.stdout:Is this ok [y/N]: 2024-04-22T21:00:35.649 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum reinstall -y kernel || true 2024-04-22T21:00:35.747 INFO:teuthology.orchestra.run.smithi150.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.7 MB/s | 255 kB 00:00 2024-04-22T21:00:35.747 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-22T21:00:35.747 INFO:teuthology.orchestra.run.smithi150.stdout:Total 1.1 MB/s | 255 kB 00:00 2024-04-22T21:00:35.769 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-22T21:00:35.940 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-22T21:00:35.940 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-22T21:00:35.945 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-22T21:00:35.945 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-22T21:00:36.081 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-22T21:00:36.144 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:00:04 ago on Mon 22 Apr 2024 09:00:32 PM UTC. 2024-04-22T21:00:36.148 INFO:teuthology.orchestra.run.smithi150.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T21:00:36.148 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T21:00:36.261 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-04-22T21:00:36.323 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-04-22T21:00:36.323 INFO:teuthology.orchestra.run.smithi130.stdout: Package Architecture Version Repository Size 2024-04-22T21:00:36.323 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-04-22T21:00:36.323 INFO:teuthology.orchestra.run.smithi130.stdout:Reinstalling: 2024-04-22T21:00:36.323 INFO:teuthology.orchestra.run.smithi130.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T21:00:36.324 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:00:36.324 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-04-22T21:00:36.324 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-04-22T21:00:36.324 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:00:36.324 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 255 k 2024-04-22T21:00:36.324 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 0 2024-04-22T21:00:36.324 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-04-22T21:00:36.791 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T21:00:36.792 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T21:00:36.998 INFO:teuthology.orchestra.run.smithi130.stdout:kernel-5.14.0-437.el9.x86_64.rpm 545 kB/s | 255 kB 00:00 2024-04-22T21:00:36.998 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-04-22T21:00:36.998 INFO:teuthology.orchestra.run.smithi130.stdout:Total 347 kB/s | 255 kB 00:00 2024-04-22T21:00:37.017 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-04-22T21:00:37.198 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-04-22T21:00:37.201 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-04-22T21:00:37.203 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-04-22T21:00:37.203 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-04-22T21:00:37.332 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-04-22T21:00:37.398 INFO:teuthology.orchestra.run.smithi130.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T21:00:37.398 INFO:teuthology.orchestra.run.smithi130.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T21:00:37.463 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T21:00:37.463 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-22T21:00:37.464 INFO:teuthology.orchestra.run.smithi150.stdout:Reinstalled: 2024-04-22T21:00:37.464 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-22T21:00:37.464 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-22T21:00:37.464 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-22T21:00:37.626 DEBUG:teuthology.orchestra.run.smithi150:> rpm -q kernel | sort -rV | head -n 1 2024-04-22T21:00:37.673 INFO:teuthology.orchestra.run.smithi150.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-22T21:00:37.673 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-22T21:00:37.673 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-22T21:00:37.673 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-22T21:00:37.674 DEBUG:teuthology.orchestra.run.smithi150:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-22T21:00:37.708 INFO:teuthology.orchestra.run.smithi150.stdout:ttyS1 2024-04-22T21:00:37.728 DEBUG:teuthology.parallel:result is None 2024-04-22T21:00:38.055 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T21:00:38.055 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T21:00:38.747 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T21:00:38.747 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:00:38.747 INFO:teuthology.orchestra.run.smithi130.stdout:Reinstalled: 2024-04-22T21:00:38.747 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-22T21:00:38.747 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:00:38.747 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-04-22T21:00:38.930 DEBUG:teuthology.orchestra.run.smithi130:> rpm -q kernel | sort -rV | head -n 1 2024-04-22T21:00:38.976 INFO:teuthology.orchestra.run.smithi130.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-22T21:00:38.976 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-22T21:00:38.977 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-22T21:00:38.977 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-22T21:00:38.977 DEBUG:teuthology.orchestra.run.smithi130:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-22T21:00:39.006 INFO:teuthology.orchestra.run.smithi130.stdout:ttyS1 2024-04-22T21:00:39.024 DEBUG:teuthology.parallel:result is None 2024-04-22T21:00:39.024 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-22T21:00:39.031 INFO:teuthology.task.internal:Creating test directory... 2024-04-22T21:00:39.032 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-22T21:00:39.066 DEBUG:teuthology.orchestra.run.smithi150:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-22T21:00:39.083 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-22T21:00:39.090 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-22T21:00:39.100 INFO:teuthology.task.internal:Creating archive directory... 2024-04-22T21:00:39.101 DEBUG:teuthology.orchestra.run.smithi130:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-22T21:00:39.120 DEBUG:teuthology.orchestra.run.smithi150:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-22T21:00:39.157 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-22T21:00:39.163 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-22T21:00:39.163 DEBUG:teuthology.orchestra.run.smithi130:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-22T21:00:39.185 DEBUG:teuthology.orchestra.run.smithi150:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-22T21:00:39.234 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T21:00:39.245 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T21:00:39.253 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T21:00:39.262 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T21:00:39.264 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-22T21:00:39.271 INFO:teuthology.task.internal:Configuring sudo... 2024-04-22T21:00:39.271 DEBUG:teuthology.orchestra.run.smithi130:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-22T21:00:39.273 DEBUG:teuthology.orchestra.run.smithi150:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-22T21:00:39.314 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-22T21:00:39.323 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-22T21:00:39.323 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-22T21:00:39.335 DEBUG:teuthology.orchestra.run.smithi150:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-22T21:00:39.370 DEBUG:teuthology.orchestra.run.smithi130:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T21:00:39.430 DEBUG:teuthology.orchestra.run.smithi130:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T21:00:39.475 DEBUG:teuthology.orchestra.run.smithi130:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T21:00:39.557 DEBUG:teuthology.orchestra.run.smithi130:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T21:00:39.594 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-22T21:00:39.594 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-22T21:00:39.658 DEBUG:teuthology.orchestra.run.smithi150:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T21:00:39.701 DEBUG:teuthology.orchestra.run.smithi150:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T21:00:39.751 DEBUG:teuthology.orchestra.run.smithi150:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T21:00:39.835 DEBUG:teuthology.orchestra.run.smithi150:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T21:00:39.879 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-22T21:00:39.879 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-22T21:00:39.946 DEBUG:teuthology.orchestra.run.smithi130:> sudo service rsyslog restart 2024-04-22T21:00:39.948 DEBUG:teuthology.orchestra.run.smithi150:> sudo service rsyslog restart 2024-04-22T21:00:39.985 INFO:teuthology.orchestra.run.smithi130.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T21:00:40.021 INFO:teuthology.orchestra.run.smithi150.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T21:00:40.400 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-22T21:00:40.408 INFO:teuthology.task.internal:Starting timer... 2024-04-22T21:00:40.409 INFO:teuthology.run_tasks:Running task pcp... 2024-04-22T21:00:40.477 INFO:teuthology.run_tasks:Running task selinux... 2024-04-22T21:00:40.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo service auditd rotate 2024-04-22T21:00:40.523 INFO:teuthology.orchestra.run.smithi130.stdout:Rotating logs: 2024-04-22T21:00:40.524 DEBUG:teuthology.orchestra.run.smithi150:> sudo service auditd rotate 2024-04-22T21:00:40.595 INFO:teuthology.orchestra.run.smithi150.stdout:Rotating logs: 2024-04-22T21:00:40.598 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-22T21:00:40.598 DEBUG:teuthology.orchestra.run.smithi130:> /usr/sbin/getenforce 2024-04-22T21:00:40.616 INFO:teuthology.orchestra.run.smithi130.stdout:Permissive 2024-04-22T21:00:40.616 DEBUG:teuthology.orchestra.run.smithi150:> /usr/sbin/getenforce 2024-04-22T21:00:40.643 INFO:teuthology.orchestra.run.smithi150.stdout:Permissive 2024-04-22T21:00:40.643 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi130.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi150.front.sepia.ceph.com': 'permissive'} 2024-04-22T21:00:40.644 DEBUG:teuthology.orchestra.run.smithi130:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-22T21:00:40.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T21:00:40.681 DEBUG:teuthology.orchestra.run.smithi150:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-22T21:00:40.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T21:00:40.709 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-22T21:00:40.709 DEBUG:teuthology.orchestra.run.smithi130:> sudo /usr/sbin/setenforce permissive 2024-04-22T21:00:40.744 DEBUG:teuthology.orchestra.run.smithi150:> sudo /usr/sbin/setenforce permissive 2024-04-22T21:00:40.773 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-22T21:00:40.783 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-22T21:00:40.789 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-22T21:00:40.971 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-22T21:00:40.987 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-22T21:00:40.988 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi130.front.sepia.ceph.com,smithi150.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-22T21:04:09.877 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi130.front.sepia.ceph.com'), Remote(name='ubuntu@smithi150.front.sepia.ceph.com')] 2024-04-22T21:04:09.879 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-04-22T21:04:09.880 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T21:04:09.954 DEBUG:teuthology.orchestra.run.smithi130:> true 2024-04-22T21:04:10.037 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-04-22T21:04:10.037 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-04-22T21:04:10.038 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T21:04:10.113 DEBUG:teuthology.orchestra.run.smithi150:> true 2024-04-22T21:04:10.189 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-04-22T21:04:10.190 INFO:teuthology.run_tasks:Running task clock... 2024-04-22T21:04:10.200 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-22T21:04:10.201 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-22T21:04:10.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T21:04:10.204 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-22T21:04:10.204 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T21:04:10.231 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-22T21:04:10.244 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-22T21:04:10.259 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-22T21:04:10.270 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: ntpd: command not found 2024-04-22T21:04:10.272 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-22T21:04:10.277 INFO:teuthology.orchestra.run.smithi130.stdout:506 Cannot talk to daemon 2024-04-22T21:04:10.290 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-22T21:04:10.295 INFO:teuthology.orchestra.run.smithi150.stderr:sudo: ntpd: command not found 2024-04-22T21:04:10.302 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-22T21:04:10.305 INFO:teuthology.orchestra.run.smithi150.stdout:506 Cannot talk to daemon 2024-04-22T21:04:10.318 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-22T21:04:10.331 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-22T21:04:10.352 INFO:teuthology.orchestra.run.smithi130.stderr:bash: line 1: ntpq: command not found 2024-04-22T21:04:10.356 INFO:teuthology.orchestra.run.smithi130.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T21:04:10.356 INFO:teuthology.orchestra.run.smithi130.stdout:=============================================================================== 2024-04-22T21:04:10.356 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T21:04:10.356 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T21:04:10.356 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T21:04:10.356 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T21:04:10.390 INFO:teuthology.orchestra.run.smithi150.stderr:bash: line 1: ntpq: command not found 2024-04-22T21:04:10.393 INFO:teuthology.orchestra.run.smithi150.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T21:04:10.393 INFO:teuthology.orchestra.run.smithi150.stdout:=============================================================================== 2024-04-22T21:04:10.393 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T21:04:10.393 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T21:04:10.393 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T21:04:10.393 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T21:04:10.394 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-22T21:04:10.505 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': '56e81df3aeb98b717efe2ab0537a8c60249f95f7'} 2024-04-22T21:04:10.505 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-22T21:04:10.506 INFO:tasks.cephadm:Cluster fsid is dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:04:10.506 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-22T21:04:10.506 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.130', 'mon.c': '[v2:172.21.15.130:3301,v1:172.21.15.130:6790]', 'mon.b': '172.21.15.150'} 2024-04-22T21:04:10.506 INFO:tasks.cephadm:First mon is mon.a on smithi130 2024-04-22T21:04:10.507 INFO:tasks.cephadm:First mgr is y 2024-04-22T21:04:10.507 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-22T21:04:10.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo hostname $(hostname -s) 2024-04-22T21:04:10.531 DEBUG:teuthology.orchestra.run.smithi150:> sudo hostname $(hostname -s) 2024-04-22T21:04:10.556 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-22T21:04:10.556 DEBUG:teuthology.orchestra.run.smithi130:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-22T21:04:10.821 INFO:teuthology.orchestra.run.smithi130.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 22 21:04 /home/ubuntu/cephtest/cephadm 2024-04-22T21:04:10.821 DEBUG:teuthology.orchestra.run.smithi150:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-22T21:04:11.052 INFO:teuthology.orchestra.run.smithi150.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 22 21:04 /home/ubuntu/cephtest/cephadm 2024-04-22T21:04:11.053 DEBUG:teuthology.orchestra.run.smithi130:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-22T21:04:11.068 DEBUG:teuthology.orchestra.run.smithi150:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-22T21:04:11.096 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-22T21:04:11.096 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-22T21:04:11.111 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-22T21:04:12.608 INFO:teuthology.orchestra.run.smithi130.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T21:04:12.688 INFO:teuthology.orchestra.run.smithi150.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T21:04:30.036 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2024-04-22T21:04:30.036 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-22T21:04:30.036 INFO:teuthology.orchestra.run.smithi150.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-22T21:04:30.036 INFO:teuthology.orchestra.run.smithi150.stdout: "repo_digests": [ 2024-04-22T21:04:30.036 INFO:teuthology.orchestra.run.smithi150.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-22T21:04:30.036 INFO:teuthology.orchestra.run.smithi150.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-22T21:04:30.036 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2024-04-22T21:04:30.036 INFO:teuthology.orchestra.run.smithi150.stdout:} 2024-04-22T21:04:34.149 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:04:34.150 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-22T21:04:34.150 INFO:teuthology.orchestra.run.smithi130.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-22T21:04:34.150 INFO:teuthology.orchestra.run.smithi130.stdout: "repo_digests": [ 2024-04-22T21:04:34.150 INFO:teuthology.orchestra.run.smithi130.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-22T21:04:34.150 INFO:teuthology.orchestra.run.smithi130.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-22T21:04:34.150 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-04-22T21:04:34.150 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:04:34.169 DEBUG:teuthology.orchestra.run.smithi130:> sudo mkdir -p /etc/ceph 2024-04-22T21:04:34.227 DEBUG:teuthology.orchestra.run.smithi150:> sudo mkdir -p /etc/ceph 2024-04-22T21:04:34.259 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 777 /etc/ceph 2024-04-22T21:04:34.283 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod 777 /etc/ceph 2024-04-22T21:04:34.326 INFO:tasks.cephadm:Writing seed config... 2024-04-22T21:04:34.327 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-04-22T21:04:34.327 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-22T21:04:34.327 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-22T21:04:34.327 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-04-22T21:04:34.327 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-22T21:04:34.327 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-22T21:04:34.327 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-22T21:04:34.328 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-22T21:04:34.328 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-22T21:04:34.328 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-22T21:04:34.328 DEBUG:teuthology.orchestra.run.smithi130:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-22T21:04:34.343 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = dd7d46b8-00eb-11ef-bc93-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-22T21:04:34.344 DEBUG:teuthology.orchestra.run.smithi130:mon.a> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.a.service 2024-04-22T21:04:34.386 DEBUG:teuthology.orchestra.run.smithi130:mgr.y> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.y.service 2024-04-22T21:04:34.429 INFO:tasks.cephadm:Bootstrapping... 2024-04-22T21:04:34.430 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.130 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-22T21:04:34.620 INFO:teuthology.orchestra.run.smithi130.stderr:-------------------------------------------------------------------------------- 2024-04-22T21:04:34.620 INFO:teuthology.orchestra.run.smithi130.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'dd7d46b8-00eb-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.130', '--skip-admin-label'] 2024-04-22T21:04:34.643 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: 4.9.4-dev 2024-04-22T21:04:34.645 INFO:teuthology.orchestra.run.smithi130.stderr:Verifying podman|docker is present... 2024-04-22T21:04:34.666 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: 4.9.4-dev 2024-04-22T21:04:34.668 INFO:teuthology.orchestra.run.smithi130.stderr:Verifying lvm2 is present... 2024-04-22T21:04:34.669 INFO:teuthology.orchestra.run.smithi130.stderr:Verifying time synchronization is in place... 2024-04-22T21:04:34.675 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-22T21:04:34.681 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: inactive 2024-04-22T21:04:34.688 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: enabled 2024-04-22T21:04:34.694 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: active 2024-04-22T21:04:34.694 INFO:teuthology.orchestra.run.smithi130.stderr:Unit chronyd.service is enabled and running 2024-04-22T21:04:34.694 INFO:teuthology.orchestra.run.smithi130.stderr:Repeating the final host check... 2024-04-22T21:04:34.716 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: 4.9.4-dev 2024-04-22T21:04:34.718 INFO:teuthology.orchestra.run.smithi130.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-22T21:04:34.718 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl is present 2024-04-22T21:04:34.718 INFO:teuthology.orchestra.run.smithi130.stderr:lvcreate is present 2024-04-22T21:04:34.724 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-22T21:04:34.730 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: inactive 2024-04-22T21:04:34.736 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: enabled 2024-04-22T21:04:34.742 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: active 2024-04-22T21:04:34.743 INFO:teuthology.orchestra.run.smithi130.stderr:Unit chronyd.service is enabled and running 2024-04-22T21:04:34.743 INFO:teuthology.orchestra.run.smithi130.stderr:Host looks OK 2024-04-22T21:04:34.743 INFO:teuthology.orchestra.run.smithi130.stderr:Cluster fsid: dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:04:34.743 INFO:teuthology.orchestra.run.smithi130.stderr:Acquiring lock 139903021895536 on /run/cephadm/dd7d46b8-00eb-11ef-bc93-c7b262605968.lock 2024-04-22T21:04:34.743 INFO:teuthology.orchestra.run.smithi130.stderr:Lock 139903021895536 acquired on /run/cephadm/dd7d46b8-00eb-11ef-bc93-c7b262605968.lock 2024-04-22T21:04:34.743 INFO:teuthology.orchestra.run.smithi130.stderr:Verifying IP 172.21.15.130 port 3300 ... 2024-04-22T21:04:34.744 INFO:teuthology.orchestra.run.smithi130.stderr:Verifying IP 172.21.15.130 port 6789 ... 2024-04-22T21:04:34.744 INFO:teuthology.orchestra.run.smithi130.stderr:Base mon IP is 172.21.15.130, final addrv is [v2:172.21.15.130:3300,v1:172.21.15.130:6789] 2024-04-22T21:04:34.746 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.130 metric 100 2024-04-22T21:04:34.746 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.130 metric 100 2024-04-22T21:04:34.748 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-22T21:04:34.749 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-04-22T21:04:34.749 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-04-22T21:04:34.751 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-22T21:04:34.751 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-22T21:04:34.752 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-22T21:04:34.752 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-22T21:04:34.752 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: inet6 fe80::b5d2:5a3e:42e4:319e/64 scope link noprefixroute 2024-04-22T21:04:34.752 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-22T21:04:34.753 INFO:teuthology.orchestra.run.smithi130.stderr:Mon IP `172.21.15.130` is in CIDR network `172.21.0.0/20` 2024-04-22T21:04:34.753 INFO:teuthology.orchestra.run.smithi130.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-22T21:04:34.753 INFO:teuthology.orchestra.run.smithi130.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T21:04:34.854 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-22T21:04:35.218 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Getting image source signatures 2024-04-22T21:04:35.218 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-22T21:04:35.219 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-22T21:04:35.219 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-22T21:04:35.219 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-22T21:04:35.219 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-22T21:04:35.219 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-22T21:04:35.394 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Writing manifest to image destination 2024-04-22T21:04:35.547 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-22T21:04:35.910 INFO:teuthology.orchestra.run.smithi130.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-22T21:04:36.372 INFO:teuthology.orchestra.run.smithi130.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-22T21:04:36.372 INFO:teuthology.orchestra.run.smithi130.stderr:Extracting ceph user uid/gid from container image... 2024-04-22T21:04:36.771 INFO:teuthology.orchestra.run.smithi130.stderr:stat: 167 167 2024-04-22T21:04:37.336 INFO:teuthology.orchestra.run.smithi130.stderr:Creating initial keys... 2024-04-22T21:04:37.752 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph-authtool: AQDl0CZmAB4fKBAA7QlO7AoU+evkk2W6mUoGrQ== 2024-04-22T21:04:41.818 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph-authtool: AQDp0CZmaHMyCBAA44FKugICOZ/Gev3TM1siTw== 2024-04-22T21:04:50.353 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph-authtool: AQDy0CZmxyz6EBAAJuoNIaDzgMi/te8rKgkbTA== 2024-04-22T21:04:50.911 INFO:teuthology.orchestra.run.smithi130.stderr:Creating initial monmap... 2024-04-22T21:04:51.317 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-22T21:04:51.318 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-22T21:04:51.318 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:04:51.318 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-22T21:04:51.758 INFO:teuthology.orchestra.run.smithi130.stderr:monmaptool for a [v2:172.21.15.130:3300,v1:172.21.15.130:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-22T21:04:51.758 INFO:teuthology.orchestra.run.smithi130.stderr:setting min_mon_release = octopus 2024-04-22T21:04:51.758 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/monmaptool: set fsid to dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:04:51.758 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-22T21:04:51.758 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-04-22T21:04:51.759 INFO:teuthology.orchestra.run.smithi130.stderr:Creating mon... 2024-04-22T21:04:52.647 INFO:teuthology.orchestra.run.smithi130.stderr:create mon.a on 2024-04-22T21:04:52.862 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-22T21:04:53.010 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968.target. 2024-04-22T21:04:53.010 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968.target. 2024-04-22T21:04:53.308 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Failed to reset failed state of unit ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.a.service: Unit ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.a.service not loaded. 2024-04-22T21:04:53.315 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Created symlink /etc/systemd/system/ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968.target.wants/ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.a.service → /etc/systemd/system/ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@.service. 2024-04-22T21:04:53.949 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:53 smithi130 podman[21502]: 2024-04-22 21:04:53.907518861 +0000 UTC m=+0.319382605 container start 45b29378c1f5686d6bf8788b39bc00a2140dbfba12defcdc522799add4136275 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, version=8, vcs-type=git, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.component=centos-stream-container, ceph=True, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD) 2024-04-22T21:04:54.012 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: disabled 2024-04-22T21:04:54.018 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: inactive 2024-04-22T21:04:54.019 INFO:teuthology.orchestra.run.smithi130.stderr:firewalld.service is not enabled 2024-04-22T21:04:54.019 INFO:teuthology.orchestra.run.smithi130.stderr:Not possible to enable service . firewalld.service is not available 2024-04-22T21:04:54.020 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for mon to start... 2024-04-22T21:04:54.020 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for mon... 2024-04-22T21:04:54.246 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:53 smithi130 ceph-mon[21520]: mkfs dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:04:54.246 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:53 smithi130 ceph-mon[21520]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T21:04:54.247 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:54 smithi130 bash[21502]: 45b29378c1f5686d6bf8788b39bc00a2140dbfba12defcdc522799add4136275 2024-04-22T21:04:54.247 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:54 smithi130 systemd[1]: Started Ceph mon.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:04:54.540 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: cluster: 2024-04-22T21:04:54.540 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: id: dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:04:54.540 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-22T21:04:54.540 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-04-22T21:04:54.541 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: services: 2024-04-22T21:04:54.541 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.59959s) 2024-04-22T21:04:54.541 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-22T21:04:54.541 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-22T21:04:54.541 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-04-22T21:04:54.541 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: data: 2024-04-22T21:04:54.541 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-22T21:04:54.541 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-22T21:04:54.541 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-22T21:04:54.541 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: pgs: 2024-04-22T21:04:54.541 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-04-22T21:04:55.097 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:54 smithi130 ceph-mon[21520]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T21:04:55.098 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:54 smithi130 ceph-mon[21520]: monmap e1: 1 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} 2024-04-22T21:04:55.098 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:54 smithi130 ceph-mon[21520]: fsmap 2024-04-22T21:04:55.098 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:54 smithi130 ceph-mon[21520]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T21:04:55.098 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:54 smithi130 ceph-mon[21520]: mgrmap e1: no daemons active 2024-04-22T21:04:55.098 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:54 smithi130 ceph-mon[21520]: from='client.? 172.21.15.130:0/1546795504' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:04:55.100 INFO:teuthology.orchestra.run.smithi130.stderr:mon is available 2024-04-22T21:04:55.100 INFO:teuthology.orchestra.run.smithi130.stderr:Assimilating anything we can from ceph.conf... 2024-04-22T21:04:55.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-04-22T21:04:55.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: [global] 2024-04-22T21:04:55.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: fsid = dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:04:55.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.130:3300,v1:172.21.15.130:6789] 2024-04-22T21:04:55.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-22T21:04:55.995 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-22T21:04:55.995 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-22T21:04:55.995 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-22T21:04:55.995 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-04-22T21:04:55.995 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: [mgr] 2024-04-22T21:04:55.995 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-04-22T21:04:55.995 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-22T21:04:55.995 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-04-22T21:04:55.995 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: [osd] 2024-04-22T21:04:55.995 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-22T21:04:55.995 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-22T21:04:56.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:56 smithi130 ceph-mon[21520]: from='client.? 172.21.15.130:0/283215427' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T21:04:56.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:56 smithi130 ceph-mon[21520]: from='client.? 172.21.15.130:0/283215427' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-22T21:04:56.556 INFO:teuthology.orchestra.run.smithi130.stderr:Generating new minimal ceph.conf... 2024-04-22T21:04:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:57 smithi130 ceph-mon[21520]: from='client.? 172.21.15.130:0/617492921' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:04:57.545 INFO:teuthology.orchestra.run.smithi130.stderr:Restarting the monitor... 2024-04-22T21:04:57.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:57 smithi130 systemd[1]: Stopping Ceph mon.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:04:57.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:57 smithi130 bash[21738]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-22T21:04:57.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:57 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a[21516]: 2024-04-22T21:04:57.700+0000 7ff2a4aeb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:04:57.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:57 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a[21516]: 2024-04-22T21:04:57.700+0000 7ff2a4aeb700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-22T21:04:58.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:57 smithi130 podman[21749]: 2024-04-22 21:04:57.803191272 +0000 UTC m=+0.202844596 container died 45b29378c1f5686d6bf8788b39bc00a2140dbfba12defcdc522799add4136275 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, version=8, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, ceph=True, GIT_CLEAN=True, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-22T21:04:58.438 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 podman[21749]: 2024-04-22 21:04:58.200572124 +0000 UTC m=+0.600225453 container cleanup 45b29378c1f5686d6bf8788b39bc00a2140dbfba12defcdc522799add4136275 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, ceph=True, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., version=8, com.redhat.component=centos-stream-container) 2024-04-22T21:04:58.439 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 bash[21749]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a 2024-04-22T21:04:58.439 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 bash[21776]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-22T21:04:58.745 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 podman[21763]: 2024-04-22 21:04:58.44317908 +0000 UTC m=+0.632198965 container remove 45b29378c1f5686d6bf8788b39bc00a2140dbfba12defcdc522799add4136275 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, architecture=x86_64, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, vendor=Red Hat, Inc., name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, distribution-scope=public, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8) 2024-04-22T21:04:58.745 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-22T21:04:58.745 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 systemd[1]: Stopped Ceph mon.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:04:58.745 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 systemd[1]: Starting Ceph mon.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:04:58.745 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 podman[21848]: 2024-04-22 21:04:58.66062027 +0000 UTC m=+0.100504591 container create e010afa4af4ddf25f78c352e5d94c39d6046b98fe1b32aa7d6a6e5138b5b993b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.openshift.tags=base centos centos-stream, version=8, RELEASE=HEAD, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vcs-type=git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8) 2024-04-22T21:04:58.745 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 podman[21848]: 2024-04-22 21:04:58.578509967 +0000 UTC m=+0.018394270 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-22T21:04:58.944 INFO:teuthology.orchestra.run.smithi130.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-22T21:04:59.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 podman[21848]: 2024-04-22 21:04:58.887973012 +0000 UTC m=+0.327857296 container init e010afa4af4ddf25f78c352e5d94c39d6046b98fe1b32aa7d6a6e5138b5b993b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, name=centos-stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, RELEASE=HEAD, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, version=8, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 podman[21848]: 2024-04-22 21:04:58.890814584 +0000 UTC m=+0.330698862 container start e010afa4af4ddf25f78c352e5d94c39d6046b98fe1b32aa7d6a6e5138b5b993b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, name=centos-stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, ceph=True, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, version=8) 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: pidfile_write: ignore empty --pid-file 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: load: jerasure load: lrc 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: RocksDB version: 6.15.5 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Compile date Apr 18 2022 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: DB SUMMARY 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: DB Session ID: 9K5R2SCVNPXNGOXO15OM 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: CURRENT file: CURRENT 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-22T21:04:59.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-22T21:04:59.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82651 ; 2024-04-22T21:04:59.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.error_if_exists: 0 2024-04-22T21:04:59.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.create_if_missing: 0 2024-04-22T21:04:59.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.paranoid_checks: 1 2024-04-22T21:04:59.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T21:04:59.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.env: 0x563dadec6860 2024-04-22T21:04:59.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.fs: Posix File System 2024-04-22T21:04:59.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.info_log: 0x563dafff7dc0 2024-04-22T21:04:59.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T21:04:59.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.statistics: (nil) 2024-04-22T21:04:59.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.use_fsync: 0 2024-04-22T21:04:59.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_log_file_size: 0 2024-04-22T21:04:59.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T21:04:59.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T21:04:59.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T21:04:59.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T21:04:59.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.allow_fallocate: 1 2024-04-22T21:04:59.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T21:04:59.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T21:04:59.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.use_direct_reads: 0 2024-04-22T21:04:59.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T21:04:59.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T21:04:59.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.db_log_dir: 2024-04-22T21:04:59.063 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-22T21:04:59.063 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T21:04:59.063 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T21:04:59.063 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T21:04:59.063 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T21:04:59.063 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T21:04:59.063 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T21:04:59.063 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T21:04:59.064 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T21:04:59.064 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.write_buffer_manager: 0x563db00e8240 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.unordered_write: 0 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T21:04:59.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T21:04:59.066 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.row_cache: None 2024-04-22T21:04:59.066 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.wal_filter: None 2024-04-22T21:04:59.066 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T21:04:59.066 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T21:04:59.066 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.preserve_deletes: 0 2024-04-22T21:04:59.066 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.two_write_queues: 0 2024-04-22T21:04:59.066 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T21:04:59.066 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.atomic_flush: 0 2024-04-22T21:04:59.066 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T21:04:59.066 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T21:04:59.066 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T21:04:59.066 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.log_readahead_size: 0 2024-04-22T21:04:59.066 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_background_jobs: 2 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_background_compactions: -1 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_subcompactions: 1 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T21:04:59.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_open_files: -1 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_background_flushes: -1 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Compression algorithms supported: 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: kZSTD supported: 0 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: kXpressCompression supported: 0 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: kLZ4Compression supported: 1 2024-04-22T21:04:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: kBZip2Compression supported: 0 2024-04-22T21:04:59.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: kZlibCompression supported: 1 2024-04-22T21:04:59.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: kSnappyCompression supported: 1 2024-04-22T21:04:59.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T21:04:59.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-22T21:04:59.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-22T21:04:59.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T21:04:59.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.merge_operator: 2024-04-22T21:04:59.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compaction_filter: None 2024-04-22T21:04:59.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T21:04:59.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T21:04:59.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T21:04:59.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T21:04:59.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563daffc3d00) 2024-04-22T21:04:59.070 INFO:journalctl@ceph.mon.a.smithi130.stdout: cache_index_and_filter_blocks: 1 2024-04-22T21:04:59.070 INFO:journalctl@ceph.mon.a.smithi130.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T21:04:59.070 INFO:journalctl@ceph.mon.a.smithi130.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T21:04:59.070 INFO:journalctl@ceph.mon.a.smithi130.stdout: pin_top_level_index_and_filter: 1 2024-04-22T21:04:59.070 INFO:journalctl@ceph.mon.a.smithi130.stdout: index_type: 0 2024-04-22T21:04:59.070 INFO:journalctl@ceph.mon.a.smithi130.stdout: data_block_index_type: 0 2024-04-22T21:04:59.070 INFO:journalctl@ceph.mon.a.smithi130.stdout: index_shortening: 1 2024-04-22T21:04:59.070 INFO:journalctl@ceph.mon.a.smithi130.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T21:04:59.070 INFO:journalctl@ceph.mon.a.smithi130.stdout: hash_index_allow_collision: 1 2024-04-22T21:04:59.070 INFO:journalctl@ceph.mon.a.smithi130.stdout: checksum: 1 2024-04-22T21:04:59.070 INFO:journalctl@ceph.mon.a.smithi130.stdout: no_block_cache: 0 2024-04-22T21:04:59.070 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache: 0x563db002e170 2024-04-22T21:04:59.070 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache_name: BinnedLRUCache 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache_options: 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: capacity : 536870912 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: num_shard_bits : 4 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: strict_capacity_limit : 0 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: high_pri_pool_ratio: 0.000 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache_compressed: (nil) 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: persistent_cache: (nil) 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_size: 4096 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_size_deviation: 10 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_restart_interval: 16 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: index_block_restart_interval: 1 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: metadata_block_size: 4096 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: partition_filters: 0 2024-04-22T21:04:59.071 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_delta_encoding: 1 2024-04-22T21:04:59.072 INFO:journalctl@ceph.mon.a.smithi130.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-22T21:04:59.072 INFO:journalctl@ceph.mon.a.smithi130.stdout: whole_key_filtering: 1 2024-04-22T21:04:59.072 INFO:journalctl@ceph.mon.a.smithi130.stdout: verify_compression: 0 2024-04-22T21:04:59.072 INFO:journalctl@ceph.mon.a.smithi130.stdout: read_amp_bytes_per_bit: 0 2024-04-22T21:04:59.072 INFO:journalctl@ceph.mon.a.smithi130.stdout: format_version: 4 2024-04-22T21:04:59.072 INFO:journalctl@ceph.mon.a.smithi130.stdout: enable_index_compression: 1 2024-04-22T21:04:59.072 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_align: 0 2024-04-22T21:04:59.072 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T21:04:59.072 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T21:04:59.072 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compression: NoCompression 2024-04-22T21:04:59.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T21:04:59.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T21:04:59.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T21:04:59.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.num_levels: 7 2024-04-22T21:04:59.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T21:04:59.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T21:04:59.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T21:04:59.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T21:04:59.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T21:04:59.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T21:04:59.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T21:04:59.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T21:04:59.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T21:04:59.074 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.arena_block_size: 4194304 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T21:04:59.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.table_properties_collectors: 2024-04-22T21:04:59.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.inplace_update_support: 0 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.bloom_locality: 0 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.max_successive_merges: 0 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.ttl: 2592000 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.enable_blob_files: false 2024-04-22T21:04:59.077 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.min_blob_size: 0 2024-04-22T21:04:59.078 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T21:04:59.078 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T21:04:59.078 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T21:04:59.078 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T21:04:59.078 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T21:04:59.078 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-22T21:04:59.078 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-22T21:04:59.078 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713819898916304, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-22T21:04:59.078 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-22T21:04:59.078 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-22T21:04:59.078 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713819898916985, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79201, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77464, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71773, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713819898, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "e1b579a8-aa90-4ad1-bc52-a43d3a590e44", "db_session_id": "9K5R2SCVNPXNGOXO15OM"}} 2024-04-22T21:04:59.078 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-22T21:04:59.078 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713819898917395, "job": 1, "event": "recovery_finished"} 2024-04-22T21:04:59.079 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T21:04:59.079 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x563db0014a80 2024-04-22T21:04:59.079 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: DB pointer 0x563db0024000 2024-04-22T21:04:59.079 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-22T21:04:59.079 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-22T21:04:59.079 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** DB Stats ** 2024-04-22T21:04:59.079 INFO:journalctl@ceph.mon.a.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:04:59.079 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T21:04:59.079 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T21:04:59.079 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T21:04:59.079 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T21:04:59.079 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: L0 2/0 79.10 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 163.1 0.00 0.00 1 0.000 0 0 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: Sum 2/0 79.10 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 163.1 0.00 0.00 1 0.000 0 0 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 163.1 0.00 0.00 1 0.000 0 0 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 163.1 0.00 0.00 1 0.000 0 0 2024-04-22T21:04:59.080 INFO:journalctl@ceph.mon.a.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:04:59.081 INFO:journalctl@ceph.mon.a.smithi130.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T21:04:59.081 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T21:04:59.081 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T21:04:59.081 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T21:04:59.081 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T21:04:59.081 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative compaction: 0.00 GB write, 25.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:04:59.081 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval compaction: 0.00 GB write, 25.50 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:04:59.081 INFO:journalctl@ceph.mon.a.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T21:04:59.081 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-04-22T21:04:59.081 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T21:04:59.081 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-04-22T21:04:59.081 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-22T21:04:59.081 INFO:journalctl@ceph.mon.a.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T21:04:59.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:04:59.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: L0 2/0 79.10 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 163.1 0.00 0.00 1 0.000 0 0 2024-04-22T21:04:59.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: Sum 2/0 79.10 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 163.1 0.00 0.00 1 0.000 0 0 2024-04-22T21:04:59.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-22T21:04:59.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-04-22T21:04:59.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-22T21:04:59.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T21:04:59.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:04:59.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 163.1 0.00 0.00 1 0.000 0 0 2024-04-22T21:04:59.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:04:59.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T21:04:59.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T21:04:59.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative compaction: 0.00 GB write, 25.19 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: starting mon.a rank 0 at public addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] at bind addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: mon.a@-1(???) e1 preinit fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: mon.a@-1(???).mds e1 new map 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: mon.a@-1(???).mds e1 print_map 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: e1 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T21:04:59.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: legacy client fscid: -1 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout: No filesystems configured 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: expand_channel_meta expand map: {default=false} 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: expand_channel_meta from 'false' to 'false' 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: expand_channel_meta expanded map: {default=false} 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: expand_channel_meta expand map: {default=info} 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: expand_channel_meta from 'info' to 'info' 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: expand_channel_meta expanded map: {default=info} 2024-04-22T21:04:59.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: expand_channel_meta expand map: {default=daemon} 2024-04-22T21:04:59.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: expand_channel_meta from 'daemon' to 'daemon' 2024-04-22T21:04:59.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: expand_channel_meta expanded map: {default=daemon} 2024-04-22T21:04:59.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: expand_channel_meta expand map: {default=debug} 2024-04-22T21:04:59.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: expand_channel_meta from 'debug' to 'debug' 2024-04-22T21:04:59.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: expand_channel_meta expanded map: {default=debug} 2024-04-22T21:04:59.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 bash[21848]: e010afa4af4ddf25f78c352e5d94c39d6046b98fe1b32aa7d6a6e5138b5b993b 2024-04-22T21:04:59.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 systemd[1]: Started Ceph mon.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:04:59.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T21:04:59.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: monmap e1: 1 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} 2024-04-22T21:04:59.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: fsmap 2024-04-22T21:04:59.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T21:04:59.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:04:58 smithi130 ceph-mon[21867]: mgrmap e1: no daemons active 2024-04-22T21:05:00.093 INFO:teuthology.orchestra.run.smithi130.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-22T21:05:00.093 INFO:teuthology.orchestra.run.smithi130.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-22T21:05:00.093 INFO:teuthology.orchestra.run.smithi130.stderr:Creating mgr... 2024-04-22T21:05:00.093 INFO:teuthology.orchestra.run.smithi130.stderr:Verifying port 9283 ... 2024-04-22T21:05:00.267 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Failed to reset failed state of unit ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.y.service: Unit ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.y.service not loaded. 2024-04-22T21:05:00.274 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Created symlink /etc/systemd/system/ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968.target.wants/ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@.service. 2024-04-22T21:05:00.437 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:00 smithi130 systemd[1]: Starting Ceph mgr.y for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:05:00.762 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:00 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3934197315' entity='client.admin' 2024-04-22T21:05:00.762 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:00 smithi130 podman[22078]: 2024-04-22 21:05:00.654694195 +0000 UTC m=+0.110809375 container create 878d76986ae8f5d27124c0276ca0fa6636a69e8699f3a5fa9655ca8d8cfda59f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_CLEAN=True, vcs-type=git, ceph=True, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, distribution-scope=public, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-22T21:05:00.763 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:00 smithi130 podman[22078]: 2024-04-22 21:05:00.562275469 +0000 UTC m=+0.018390661 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-22T21:05:00.989 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: disabled 2024-04-22T21:05:00.996 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: inactive 2024-04-22T21:05:00.997 INFO:teuthology.orchestra.run.smithi130.stderr:firewalld.service is not enabled 2024-04-22T21:05:00.997 INFO:teuthology.orchestra.run.smithi130.stderr:Not possible to enable service . firewalld.service is not available 2024-04-22T21:05:01.005 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: disabled 2024-04-22T21:05:01.012 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: inactive 2024-04-22T21:05:01.013 INFO:teuthology.orchestra.run.smithi130.stderr:firewalld.service is not enabled 2024-04-22T21:05:01.013 INFO:teuthology.orchestra.run.smithi130.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-22T21:05:01.013 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for mgr to start... 2024-04-22T21:05:01.013 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for mgr... 2024-04-22T21:05:01.139 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:00 smithi130 podman[22078]: 2024-04-22 21:05:00.882443265 +0000 UTC m=+0.338558445 container init 878d76986ae8f5d27124c0276ca0fa6636a69e8699f3a5fa9655ca8d8cfda59f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y, architecture=x86_64, ceph=True, io.openshift.tags=base centos centos-stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , name=centos-stream, release=754, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, version=8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_BRANCH=HEAD) 2024-04-22T21:05:01.139 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:00 smithi130 podman[22078]: 2024-04-22 21:05:00.885177636 +0000 UTC m=+0.341292816 container start 878d76986ae8f5d27124c0276ca0fa6636a69e8699f3a5fa9655ca8d8cfda59f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_CLEAN=True, vcs-type=git, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, RELEASE=HEAD, name=centos-stream, ceph=True, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-22T21:05:01.139 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:00 smithi130 bash[22078]: 878d76986ae8f5d27124c0276ca0fa6636a69e8699f3a5fa9655ca8d8cfda59f 2024-04-22T21:05:01.139 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:00 smithi130 systemd[1]: Started Ceph mgr.y for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:05:01.139 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:01 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:01.027+0000 7f7b6ba8c000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:05:01.392 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:01 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:01.162+0000 7f7b6ba8c000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:05:01.392 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:01 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:01.260+0000 7f7b6ba8c000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:05:01.568 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-04-22T21:05:01.568 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-04-22T21:05:01.568 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsid": "dd7d46b8-00eb-11ef-bc93-c7b262605968", 2024-04-22T21:05:01.568 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "health": { 2024-04-22T21:05:01.568 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T21:05:01.568 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T21:05:01.568 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 0 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "a" 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:01.569 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T21:05:01.570 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:01.570 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T21:05:01.570 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T21:05:01.570 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T21:05:01.570 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T21:05:01.573 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T21:05:01.573 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T21:05:01.573 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T21:05:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "available": false, 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "iostat", 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "nfs", 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "restful" 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modified": "2024-04-22T21:04:53.945766+0000", 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-04-22T21:05:01.575 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:01.576 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T21:05:01.576 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-04-22T21:05:01.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:01 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2078182910' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T21:05:01.664 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:01 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:01.581+0000 7f7b6ba8c000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:05:01.664 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:01 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:01.667+0000 7f7b6ba8c000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:05:02.161 INFO:teuthology.orchestra.run.smithi130.stderr:mgr not available, waiting (1/15)... 2024-04-22T21:05:03.059 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:02.797+0000 7f7b6ba8c000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:05:03.059 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:02.957+0000 7f7b6ba8c000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:05:03.059 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:03.036+0000 7f7b6ba8c000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:05:03.757 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:03.345+0000 7f7b6ba8c000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:05:03.758 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:03.433+0000 7f7b6ba8c000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:05:03.758 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:03.523+0000 7f7b6ba8c000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:05:04.059 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:03.761+0000 7f7b6ba8c000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:05:04.503 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:04 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:04.196+0000 7f7b6ba8c000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:05:04.682 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-04-22T21:05:04.682 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-04-22T21:05:04.682 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsid": "dd7d46b8-00eb-11ef-bc93-c7b262605968", 2024-04-22T21:05:04.682 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "health": { 2024-04-22T21:05:04.682 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T21:05:04.682 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T21:05:04.682 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 0 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "a" 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:04.683 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T21:05:04.684 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "available": false, 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "iostat", 2024-04-22T21:05:04.685 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "nfs", 2024-04-22T21:05:04.686 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "restful" 2024-04-22T21:05:04.686 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-04-22T21:05:04.686 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-04-22T21:05:04.686 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:04.686 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T21:05:04.686 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:04.686 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modified": "2024-04-22T21:04:53.945766+0000", 2024-04-22T21:05:04.686 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-04-22T21:05:04.686 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:04.686 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T21:05:04.686 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-04-22T21:05:04.787 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:04 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:04.692+0000 7f7b6ba8c000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:05:04.787 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:04 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:04.791+0000 7f7b6ba8c000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:05:05.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:04 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1592184615' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T21:05:05.276 INFO:teuthology.orchestra.run.smithi130.stderr:mgr not available, waiting (2/15)... 2024-04-22T21:05:05.559 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:05.084+0000 7f7b6ba8c000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:05:05.559 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:05.171+0000 7f7b6ba8c000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:05:05.920 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:05.584+0000 7f7b6ba8c000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:05:05.920 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:05.722+0000 7f7b6ba8c000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:05:05.920 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:05.812+0000 7f7b6ba8c000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:05:05.920 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:05.923+0000 7f7b6ba8c000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:05:06.309 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:06.010+0000 7f7b6ba8c000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:05:06.809 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:06.545+0000 7f7b6ba8c000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:05:07.309 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:06.902+0000 7f7b6ba8c000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:05:07.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-mon[21867]: Activating manager daemon y 2024-04-22T21:05:07.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-mon[21867]: mgrmap e2: y(active, starting, since 0.00263134s) 2024-04-22T21:05:07.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-mon[21867]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:05:07.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-mon[21867]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:05:07.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-mon[21867]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:05:07.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-mon[21867]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:05:07.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-mon[21867]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:05:07.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-mon[21867]: Manager daemon y is now available 2024-04-22T21:05:07.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-mon[21867]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:05:07.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-mon[21867]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:05:07.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-mon[21867]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' 2024-04-22T21:05:07.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-mon[21867]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' 2024-04-22T21:05:07.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:06 smithi130 ceph-mon[21867]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' 2024-04-22T21:05:07.732 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-04-22T21:05:07.732 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-04-22T21:05:07.732 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsid": "dd7d46b8-00eb-11ef-bc93-c7b262605968", 2024-04-22T21:05:07.732 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "health": { 2024-04-22T21:05:07.732 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T21:05:07.732 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 0 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "a" 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T21:05:07.733 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T21:05:07.734 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "available": false, 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T21:05:07.735 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T21:05:07.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "iostat", 2024-04-22T21:05:07.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "nfs", 2024-04-22T21:05:07.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "restful" 2024-04-22T21:05:07.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-04-22T21:05:07.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-04-22T21:05:07.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:07.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T21:05:07.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:07.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modified": "2024-04-22T21:04:53.945766+0000", 2024-04-22T21:05:07.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-04-22T21:05:07.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:07.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T21:05:07.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-04-22T21:05:08.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:07 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2034743182' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T21:05:08.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:07 smithi130 ceph-mon[21867]: mgrmap e3: y(active, since 1.00566s) 2024-04-22T21:05:08.315 INFO:teuthology.orchestra.run.smithi130.stderr:mgr not available, waiting (3/15)... 2024-04-22T21:05:10.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:09 smithi130 ceph-mon[21867]: mgrmap e4: y(active, since 2s) 2024-04-22T21:05:10.992 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-04-22T21:05:10.993 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-04-22T21:05:10.993 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsid": "dd7d46b8-00eb-11ef-bc93-c7b262605968", 2024-04-22T21:05:10.993 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "health": { 2024-04-22T21:05:10.993 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T21:05:10.993 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T21:05:10.993 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T21:05:10.993 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:10.993 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T21:05:10.993 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T21:05:10.993 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 0 2024-04-22T21:05:10.993 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-04-22T21:05:10.993 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T21:05:10.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "a" 2024-04-22T21:05:10.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-04-22T21:05:10.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-04-22T21:05:10.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T21:05:10.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:10.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T21:05:10.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T21:05:10.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:10.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T21:05:10.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:10.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T21:05:10.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T21:05:10.994 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T21:05:10.998 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T21:05:10.998 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T21:05:10.998 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T21:05:10.998 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:10.998 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T21:05:10.998 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T21:05:10.998 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T21:05:10.998 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T21:05:10.998 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "available": true, 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T21:05:10.999 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "iostat", 2024-04-22T21:05:11.000 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "nfs", 2024-04-22T21:05:11.000 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "restful" 2024-04-22T21:05:11.000 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-04-22T21:05:11.000 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-04-22T21:05:11.000 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:11.000 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T21:05:11.000 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T21:05:11.000 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modified": "2024-04-22T21:04:53.945766+0000", 2024-04-22T21:05:11.000 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-04-22T21:05:11.000 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-04-22T21:05:11.000 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T21:05:11.000 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-04-22T21:05:11.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:11 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3742667986' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T21:05:11.605 INFO:teuthology.orchestra.run.smithi130.stderr:mgr is available 2024-04-22T21:05:12.182 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-04-22T21:05:12.182 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: [global] 2024-04-22T21:05:12.182 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: fsid = dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:05:12.183 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-22T21:05:12.183 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-22T21:05:12.183 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-22T21:05:12.183 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-22T21:05:12.183 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-04-22T21:05:12.183 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: [mgr] 2024-04-22T21:05:12.184 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-22T21:05:12.184 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-04-22T21:05:12.184 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: [osd] 2024-04-22T21:05:12.184 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-22T21:05:12.184 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-22T21:05:12.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:12 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3876330428' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T21:05:12.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:12 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3876330428' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-22T21:05:12.670 INFO:teuthology.orchestra.run.smithi130.stderr:Enabling cephadm module... 2024-04-22T21:05:13.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:13 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1432866693' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-22T21:05:13.677 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ignoring --setuser ceph since I am not root 2024-04-22T21:05:13.677 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ignoring --setgroup ceph since I am not root 2024-04-22T21:05:13.677 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:13.551+0000 7f990e3fb000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:05:13.677 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:13.680+0000 7f990e3fb000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:05:13.981 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:13.778+0000 7f990e3fb000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:05:14.309 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:14.089+0000 7f990e3fb000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:05:14.309 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:14.173+0000 7f990e3fb000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:05:14.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:14 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1432866693' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-22T21:05:14.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:14 smithi130 ceph-mon[21867]: mgrmap e5: y(active, since 6s) 2024-04-22T21:05:14.855 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-04-22T21:05:14.855 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-22T21:05:14.855 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "available": true, 2024-04-22T21:05:14.855 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-22T21:05:14.855 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-22T21:05:14.855 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-04-22T21:05:15.405 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:15 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4080410037' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T21:05:15.405 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:15 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:15.307+0000 7f990e3fb000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:05:15.407 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for the mgr to restart... 2024-04-22T21:05:15.408 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for mgr epoch 5... 2024-04-22T21:05:15.691 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:15 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:15.477+0000 7f990e3fb000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:05:15.691 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:15 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:15.560+0000 7f990e3fb000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:05:15.967 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:15 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:15.881+0000 7f990e3fb000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:05:15.967 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:15 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:15.971+0000 7f990e3fb000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:05:16.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:16 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:16.063+0000 7f990e3fb000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:05:16.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:16 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:16.303+0000 7f990e3fb000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:05:17.015 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:16 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:16.744+0000 7f990e3fb000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:05:17.309 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:17 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:17.253+0000 7f990e3fb000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:05:17.640 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:17 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:17.347+0000 7f990e3fb000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:05:18.059 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:17 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:17.644+0000 7f990e3fb000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:05:18.059 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:17 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:17.734+0000 7f990e3fb000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:05:18.468 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:18.138+0000 7f990e3fb000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:05:18.468 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:18.269+0000 7f990e3fb000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:05:18.469 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:18.360+0000 7f990e3fb000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:05:18.469 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:18.472+0000 7f990e3fb000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:05:18.809 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:18.561+0000 7f990e3fb000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:05:19.458 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:19 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:19.103+0000 7f990e3fb000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:05:19.809 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:19 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:19.462+0000 7f990e3fb000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:05:20.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: Active manager daemon y restarted 2024-04-22T21:05:20.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: Activating manager daemon y 2024-04-22T21:05:20.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: osdmap e2: 0 total, 0 up, 0 in 2024-04-22T21:05:20.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: mgrmap e6: y(active, starting, since 0.462448s) 2024-04-22T21:05:20.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:05:20.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:05:20.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:05:20.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:05:20.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:05:20.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: Manager daemon y is now available 2024-04-22T21:05:20.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:05:20.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:05:20.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:05:20.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:05:20.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:05:20.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:05:20.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:20 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:05:20.932 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-04-22T21:05:20.932 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-22T21:05:20.932 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "initialized": true 2024-04-22T21:05:20.932 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-04-22T21:05:21.545 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:21 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:05:21] ENGINE Bus STARTING 2024-04-22T21:05:21.545 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:21 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:05:21] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:05:21.545 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:21 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:05:21] ENGINE Bus STARTED 2024-04-22T21:05:21.547 INFO:teuthology.orchestra.run.smithi130.stderr:mgr epoch 5 is available 2024-04-22T21:05:21.547 INFO:teuthology.orchestra.run.smithi130.stderr:Setting orchestrator backend to cephadm... 2024-04-22T21:05:22.220 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:21 smithi130 ceph-mon[21867]: mgrmap e7: y(active, since 1.46492s) 2024-04-22T21:05:22.220 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:21 smithi130 ceph-mon[21867]: [22/Apr/2024:21:05:21] ENGINE Bus STARTING 2024-04-22T21:05:22.220 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:21 smithi130 ceph-mon[21867]: [22/Apr/2024:21:05:21] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:05:22.220 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:21 smithi130 ceph-mon[21867]: [22/Apr/2024:21:05:21] ENGINE Bus STARTED 2024-04-22T21:05:22.221 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:21 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:05:22.221 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:21 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:05:23.387 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:23 smithi130 ceph-mon[21867]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:05:23.387 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:23 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:05:23.387 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:23 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:05:23.387 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:23 smithi130 ceph-mon[21867]: mgrmap e8: y(active, since 2s) 2024-04-22T21:05:23.388 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: value unchanged 2024-04-22T21:05:23.960 INFO:teuthology.orchestra.run.smithi130.stderr:Generating ssh key... 2024-04-22T21:05:24.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-mon[21867]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:05:25.059 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: Generating public/private rsa key pair. 2024-04-22T21:05:25.059 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: Your identification has been saved in /tmp/tmpdk2uc2wv/key. 2024-04-22T21:05:25.059 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: Your public key has been saved in /tmp/tmpdk2uc2wv/key.pub. 2024-04-22T21:05:25.059 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: The key fingerprint is: 2024-04-22T21:05:25.059 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: SHA256:5WmM+mq8sLSHed8Ancjx/5UjoANoZd3HNBeZ+ujRL8w ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:05:25.060 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: The key's randomart image is: 2024-04-22T21:05:25.060 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: +---[RSA 3072]----+ 2024-04-22T21:05:25.060 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: | o o+ | 2024-04-22T21:05:25.060 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: | . . o oo | 2024-04-22T21:05:25.060 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: | + . o o. | 2024-04-22T21:05:25.060 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: | = = * o. | 2024-04-22T21:05:25.060 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: | o = S * + | 2024-04-22T21:05:25.060 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: | . + + .o o. | 2024-04-22T21:05:25.060 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: | o+. + ...++. | 2024-04-22T21:05:25.060 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: | .o+=. + ..oE..| 2024-04-22T21:05:25.060 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: | o+o+o . . . | 2024-04-22T21:05:25.060 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: +----[SHA256]-----+ 2024-04-22T21:05:25.974 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:25 smithi130 ceph-mon[21867]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:05:25.975 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:25 smithi130 ceph-mon[21867]: Generating ssh key... 2024-04-22T21:05:25.975 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:25 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:05:25.975 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:25 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:05:25.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuhuueG5Oblx1SiNzo8WCQJu6TKqbDAZiMzFC6+rd8Wqq9FUdZJu9cz5gH4s7Nve+7Mz4uiYdl/gjjSmerLorxOYrZYeY3XkAipjJAqecYVPfMjT4qDcd47YQgGT0WMhRfGeE3fXXubUW9OSz1DiYT3/tm8piir2ZzBWT3xydWhZruQn8sbph3/+kRsdaNbEqDE4hipbOuqvddgHhey7cHt6FkJm7PQXNNV5ehEaQ+6/Nte/tyNBVAN1jDQQO4TXNSGvfLvx6heWl11B4c/xMB1tqIUFf7xmFUgW1FknvVd6vqt2QD+6r3q33e7bVPkwX6K7Pstan865OMZndN69LhikNllUZNZJs4pvg/1MXWOVSQeZ2GkKXcnGONcs8OdsHAL2BqSG72GCG1BMdAtRXSJJHJRj/fWd9l/bzEp5TITU+KVf0WtsTrrnx0Aqvz4KyoRrBBFoJesB1h63czI8udSpO6Eu4Vvim762cx3ihGXso2RqOK3+ZIvdz+A3s2Q7U= ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:05:26.548 INFO:teuthology.orchestra.run.smithi130.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-22T21:05:26.549 INFO:teuthology.orchestra.run.smithi130.stderr:Adding key to root@localhost authorized_keys... 2024-04-22T21:05:26.549 INFO:teuthology.orchestra.run.smithi130.stderr:Adding host smithi130... 2024-04-22T21:05:27.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:26 smithi130 ceph-mon[21867]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:05:28.006 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: Added host 'smithi130' with addr '172.21.15.130' 2024-04-22T21:05:28.041 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:27 smithi130 ceph-mon[21867]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi130", "addr": "172.21.15.130", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:05:28.580 INFO:teuthology.orchestra.run.smithi130.stderr:Deploying unmanaged mon service... 2024-04-22T21:05:29.197 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:29 smithi130 ceph-mon[21867]: Deploying cephadm binary to smithi130 2024-04-22T21:05:29.197 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:29 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:05:29.197 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:29 smithi130 ceph-mon[21867]: Added host smithi130 2024-04-22T21:05:29.197 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:29 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:05:29.197 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-22T21:05:30.014 INFO:teuthology.orchestra.run.smithi130.stderr:Deploying unmanaged mgr service... 2024-04-22T21:05:30.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:30 smithi130 ceph-mon[21867]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:05:30.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:30 smithi130 ceph-mon[21867]: Saving service mon spec with placement count:5 2024-04-22T21:05:30.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:30 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:05:30.600 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-22T21:05:31.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:31 smithi130 ceph-mon[21867]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:05:31.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:31 smithi130 ceph-mon[21867]: Saving service mgr spec with placement count:2 2024-04-22T21:05:31.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:31 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:05:33.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:32 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2420067331' entity='client.admin' 2024-04-22T21:05:33.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:32 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:05:34.294 INFO:teuthology.orchestra.run.smithi130.stderr:Enabling the dashboard module... 2024-04-22T21:05:34.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:34 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:05:34.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:34 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3732946745' entity='client.admin' 2024-04-22T21:05:35.776 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:35 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1155279535' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-22T21:05:35.777 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:35 smithi130 ceph-mon[21867]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:05:35.777 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ignoring --setuser ceph since I am not root 2024-04-22T21:05:35.777 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ignoring --setgroup ceph since I am not root 2024-04-22T21:05:36.042 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:35.817+0000 7f2c914ee000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:05:36.043 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:35.947+0000 7f2c914ee000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:05:36.043 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:36.046+0000 7f2c914ee000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:05:36.620 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:36.361+0000 7f2c914ee000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:05:36.620 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:36.444+0000 7f2c914ee000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:05:36.985 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:36 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1155279535' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-22T21:05:36.986 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:36 smithi130 ceph-mon[21867]: mgrmap e9: y(active, since 16s) 2024-04-22T21:05:36.987 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-04-22T21:05:36.987 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-22T21:05:36.987 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "available": true, 2024-04-22T21:05:36.987 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-22T21:05:36.987 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-22T21:05:36.987 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-04-22T21:05:37.601 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for the mgr to restart... 2024-04-22T21:05:37.601 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for mgr epoch 9... 2024-04-22T21:05:37.836 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:37 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3562640159' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T21:05:37.836 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:37 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:37.584+0000 7f2c914ee000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:05:37.836 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:37 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:37.757+0000 7f2c914ee000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:05:37.836 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:37 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:37.840+0000 7f2c914ee000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:05:38.559 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:38.158+0000 7f2c914ee000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:05:38.559 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:38.248+0000 7f2c914ee000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:05:38.559 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:38.341+0000 7f2c914ee000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:05:39.007 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:38.580+0000 7f2c914ee000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:05:39.309 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:39.011+0000 7f2c914ee000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:05:39.809 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:39.514+0000 7f2c914ee000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:05:39.809 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:39.607+0000 7f2c914ee000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:05:40.309 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:39.898+0000 7f2c914ee000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:05:40.309 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:39.985+0000 7f2c914ee000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:05:40.714 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:40.385+0000 7f2c914ee000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:05:40.714 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:40.514+0000 7f2c914ee000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:05:40.714 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:40.606+0000 7f2c914ee000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:05:40.715 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:40.718+0000 7f2c914ee000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:05:41.059 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:40.806+0000 7f2c914ee000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:05:41.696 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:41 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:41.342+0000 7f2c914ee000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:05:41.981 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:41 smithi130 ceph-mon[21867]: Active manager daemon y restarted 2024-04-22T21:05:41.982 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:41 smithi130 ceph-mon[21867]: Activating manager daemon y 2024-04-22T21:05:41.982 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:41 smithi130 ceph-mon[21867]: osdmap e3: 0 total, 0 up, 0 in 2024-04-22T21:05:41.982 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:41 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:05:41.700+0000 7f2c914ee000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:05:41.982 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:41 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:05:41] ENGINE Bus STARTING 2024-04-22T21:05:42.309 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:05:42] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:05:42.309 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:05:42] ENGINE Bus STARTED 2024-04-22T21:05:42.766 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-04-22T21:05:42.766 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-22T21:05:42.766 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "initialized": true 2024-04-22T21:05:42.766 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-04-22T21:05:43.017 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: mgrmap e10: y(active, starting, since 0.0524932s) 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: Manager daemon y is now available 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: [22/Apr/2024:21:05:41] ENGINE Bus STARTING 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: [22/Apr/2024:21:05:42] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: [22/Apr/2024:21:05:42] ENGINE Bus STARTED 2024-04-22T21:05:43.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:05:43.452 INFO:teuthology.orchestra.run.smithi130.stderr:mgr epoch 9 is available 2024-04-22T21:05:43.452 INFO:teuthology.orchestra.run.smithi130.stderr:Generating a dashboard self-signed certificate... 2024-04-22T21:05:43.982 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:43 smithi130 ceph-mon[21867]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-22T21:05:43.982 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:43 smithi130 ceph-mon[21867]: mgrmap e11: y(active, since 1.06035s) 2024-04-22T21:05:43.983 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:43 smithi130 ceph-mon[21867]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-22T21:05:43.983 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:43 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:05:44.324 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-22T21:05:45.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:44 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:05:45.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:44 smithi130 ceph-mon[21867]: mgrmap e12: y(active, since 2s) 2024-04-22T21:05:45.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:44 smithi130 ceph-mon[21867]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:05:45.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:44 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:05:45.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:44 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:05:45.370 INFO:teuthology.orchestra.run.smithi130.stderr:Creating initial admin user... 2024-04-22T21:05:46.215 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$k/YVcBDonavqZNpZRwIuFuxNpNZy6fRDeek0prJm3wY5iGLdbDPrO", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1713819946, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-22T21:05:46.880 INFO:teuthology.orchestra.run.smithi130.stderr:Fetching dashboard port number... 2024-04-22T21:05:47.406 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:47 smithi130 ceph-mon[21867]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:05:47.406 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:47 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:05:47.565 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 8443 2024-04-22T21:05:48.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:48 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4199589592' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-22T21:05:48.427 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: disabled 2024-04-22T21:05:48.434 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: inactive 2024-04-22T21:05:48.435 INFO:teuthology.orchestra.run.smithi130.stderr:firewalld.service is not enabled 2024-04-22T21:05:48.435 INFO:teuthology.orchestra.run.smithi130.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-22T21:05:48.437 INFO:teuthology.orchestra.run.smithi130.stderr:Ceph Dashboard is now available at: 2024-04-22T21:05:48.437 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-04-22T21:05:48.437 INFO:teuthology.orchestra.run.smithi130.stderr: URL: https://smithi130:8443/ 2024-04-22T21:05:48.437 INFO:teuthology.orchestra.run.smithi130.stderr: User: admin 2024-04-22T21:05:48.437 INFO:teuthology.orchestra.run.smithi130.stderr: Password: tahawq7y4h 2024-04-22T21:05:48.437 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-04-22T21:05:48.437 INFO:teuthology.orchestra.run.smithi130.stderr:Enabling autotune for osd_memory_target 2024-04-22T21:05:49.371 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:49 smithi130 ceph-mon[21867]: mgrmap e13: y(active, since 6s) 2024-04-22T21:05:50.408 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-22T21:05:50.998 INFO:teuthology.orchestra.run.smithi130.stderr:You can access the Ceph CLI with: 2024-04-22T21:05:50.998 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-04-22T21:05:50.998 INFO:teuthology.orchestra.run.smithi130.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-22T21:05:50.998 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-04-22T21:05:50.998 INFO:teuthology.orchestra.run.smithi130.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-22T21:05:50.998 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-04-22T21:05:50.998 INFO:teuthology.orchestra.run.smithi130.stderr: ceph telemetry on 2024-04-22T21:05:50.998 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-04-22T21:05:50.998 INFO:teuthology.orchestra.run.smithi130.stderr:For more information see: 2024-04-22T21:05:50.998 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-04-22T21:05:50.998 INFO:teuthology.orchestra.run.smithi130.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-22T21:05:50.999 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-04-22T21:05:50.999 INFO:teuthology.orchestra.run.smithi130.stderr:Bootstrap complete. 2024-04-22T21:05:51.032 INFO:tasks.cephadm:Fetching config... 2024-04-22T21:05:51.032 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-22T21:05:51.032 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-22T21:05:51.048 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-22T21:05:51.048 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-22T21:05:51.048 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-22T21:05:51.103 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-22T21:05:51.103 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-22T21:05:51.104 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-22T21:05:51.167 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-22T21:05:51.167 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-22T21:05:51.167 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-22T21:05:51.222 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-22T21:05:51.223 DEBUG:teuthology.orchestra.run.smithi130:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuhuueG5Oblx1SiNzo8WCQJu6TKqbDAZiMzFC6+rd8Wqq9FUdZJu9cz5gH4s7Nve+7Mz4uiYdl/gjjSmerLorxOYrZYeY3XkAipjJAqecYVPfMjT4qDcd47YQgGT0WMhRfGeE3fXXubUW9OSz1DiYT3/tm8piir2ZzBWT3xydWhZruQn8sbph3/+kRsdaNbEqDE4hipbOuqvddgHhey7cHt6FkJm7PQXNNV5ehEaQ+6/Nte/tyNBVAN1jDQQO4TXNSGvfLvx6heWl11B4c/xMB1tqIUFf7xmFUgW1FknvVd6vqt2QD+6r3q33e7bVPkwX6K7Pstan865OMZndN69LhikNllUZNZJs4pvg/1MXWOVSQeZ2GkKXcnGONcs8OdsHAL2BqSG72GCG1BMdAtRXSJJHJRj/fWd9l/bzEp5TITU+KVf0WtsTrrnx0Aqvz4KyoRrBBFoJesB1h63czI8udSpO6Eu4Vvim762cx3ihGXso2RqOK3+ZIvdz+A3s2Q7U= ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-22T21:05:51.294 INFO:teuthology.orchestra.run.smithi130.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuhuueG5Oblx1SiNzo8WCQJu6TKqbDAZiMzFC6+rd8Wqq9FUdZJu9cz5gH4s7Nve+7Mz4uiYdl/gjjSmerLorxOYrZYeY3XkAipjJAqecYVPfMjT4qDcd47YQgGT0WMhRfGeE3fXXubUW9OSz1DiYT3/tm8piir2ZzBWT3xydWhZruQn8sbph3/+kRsdaNbEqDE4hipbOuqvddgHhey7cHt6FkJm7PQXNNV5ehEaQ+6/Nte/tyNBVAN1jDQQO4TXNSGvfLvx6heWl11B4c/xMB1tqIUFf7xmFUgW1FknvVd6vqt2QD+6r3q33e7bVPkwX6K7Pstan865OMZndN69LhikNllUZNZJs4pvg/1MXWOVSQeZ2GkKXcnGONcs8OdsHAL2BqSG72GCG1BMdAtRXSJJHJRj/fWd9l/bzEp5TITU+KVf0WtsTrrnx0Aqvz4KyoRrBBFoJesB1h63czI8udSpO6Eu4Vvim762cx3ihGXso2RqOK3+ZIvdz+A3s2Q7U= ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:05:51.304 DEBUG:teuthology.orchestra.run.smithi150:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuhuueG5Oblx1SiNzo8WCQJu6TKqbDAZiMzFC6+rd8Wqq9FUdZJu9cz5gH4s7Nve+7Mz4uiYdl/gjjSmerLorxOYrZYeY3XkAipjJAqecYVPfMjT4qDcd47YQgGT0WMhRfGeE3fXXubUW9OSz1DiYT3/tm8piir2ZzBWT3xydWhZruQn8sbph3/+kRsdaNbEqDE4hipbOuqvddgHhey7cHt6FkJm7PQXNNV5ehEaQ+6/Nte/tyNBVAN1jDQQO4TXNSGvfLvx6heWl11B4c/xMB1tqIUFf7xmFUgW1FknvVd6vqt2QD+6r3q33e7bVPkwX6K7Pstan865OMZndN69LhikNllUZNZJs4pvg/1MXWOVSQeZ2GkKXcnGONcs8OdsHAL2BqSG72GCG1BMdAtRXSJJHJRj/fWd9l/bzEp5TITU+KVf0WtsTrrnx0Aqvz4KyoRrBBFoJesB1h63czI8udSpO6Eu4Vvim762cx3ihGXso2RqOK3+ZIvdz+A3s2Q7U= ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-22T21:05:51.345 INFO:teuthology.orchestra.run.smithi150.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuhuueG5Oblx1SiNzo8WCQJu6TKqbDAZiMzFC6+rd8Wqq9FUdZJu9cz5gH4s7Nve+7Mz4uiYdl/gjjSmerLorxOYrZYeY3XkAipjJAqecYVPfMjT4qDcd47YQgGT0WMhRfGeE3fXXubUW9OSz1DiYT3/tm8piir2ZzBWT3xydWhZruQn8sbph3/+kRsdaNbEqDE4hipbOuqvddgHhey7cHt6FkJm7PQXNNV5ehEaQ+6/Nte/tyNBVAN1jDQQO4TXNSGvfLvx6heWl11B4c/xMB1tqIUFf7xmFUgW1FknvVd6vqt2QD+6r3q33e7bVPkwX6K7Pstan865OMZndN69LhikNllUZNZJs4pvg/1MXWOVSQeZ2GkKXcnGONcs8OdsHAL2BqSG72GCG1BMdAtRXSJJHJRj/fWd9l/bzEp5TITU+KVf0WtsTrrnx0Aqvz4KyoRrBBFoJesB1h63czI8udSpO6Eu4Vvim762cx3ihGXso2RqOK3+ZIvdz+A3s2Q7U= ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:05:51.357 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-22T21:05:51.424 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:51 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3768518882' entity='client.admin' 2024-04-22T21:05:54.330 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-22T21:05:54.330 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-22T21:05:54.871 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:54 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/714300523' entity='client.admin' 2024-04-22T21:05:54.871 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:54 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:05:54.871 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:54 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi130", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:05:54.871 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:54 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:05:54.871 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:54 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:05:54.871 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:54 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:05:56.988 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi150 2024-04-22T21:05:56.989 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-22T21:05:56.989 DEBUG:teuthology.orchestra.run.smithi150:> dd of=/etc/ceph/ceph.conf 2024-04-22T21:05:57.007 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-22T21:05:57.007 DEBUG:teuthology.orchestra.run.smithi150:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:05:57.064 INFO:tasks.cephadm:Adding host smithi150 to orchestrator... 2024-04-22T21:05:57.064 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch host add smithi150 2024-04-22T21:05:57.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:57 smithi130 ceph-mon[21867]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:05:57.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:05:57.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:05:57.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:05:57.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:05:57.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:57 smithi130 ceph-mon[21867]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:05:57.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:57 smithi130 ceph-mon[21867]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:05:57.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:05:57.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:05:59.962 INFO:teuthology.orchestra.run.smithi130.stdout:Added host 'smithi150' with addr '172.21.15.150' 2024-04-22T21:06:00.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:59 smithi130 ceph-mon[21867]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi150", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:00.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:05:59 smithi130 ceph-mon[21867]: Deploying cephadm binary to smithi150 2024-04-22T21:06:00.622 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-22T21:06:01.201 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:00 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:01.201 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:00 smithi130 ceph-mon[21867]: Added host smithi150 2024-04-22T21:06:01.201 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:00 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:01.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:00 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:01.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:00 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:01.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:00 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:01.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:00 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:02.471 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:06:02.471 INFO:teuthology.orchestra.run.smithi130.stdout:[{"addr": "172.21.15.130", "hostname": "smithi130", "labels": [], "status": ""}, {"addr": "172.21.15.150", "hostname": "smithi150", "labels": [], "status": ""}] 2024-04-22T21:06:03.056 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:02 smithi130 ceph-mon[21867]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:03.056 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:02 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:03.056 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:02 smithi130 ceph-mon[21867]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T21:06:03.084 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-22T21:06:03.084 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-22T21:06:05.555 INFO:teuthology.orchestra.run.smithi130.stderr:adjusted tunables profile to default 2024-04-22T21:06:05.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:05 smithi130 ceph-mon[21867]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:05.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:05 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:05.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:05 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:05.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:05 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi150", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:06:05.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:05 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:05.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:05 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3577576924' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-22T21:06:06.169 INFO:tasks.cephadm:Adding mon.a on smithi130 2024-04-22T21:06:06.169 INFO:tasks.cephadm:Adding mon.c on smithi130 2024-04-22T21:06:06.169 INFO:tasks.cephadm:Adding mon.b on smithi150 2024-04-22T21:06:06.170 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch apply mon '3;smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi150:172.21.15.150=b' 2024-04-22T21:06:06.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:06 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3577576924' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-22T21:06:06.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:06 smithi130 ceph-mon[21867]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T21:06:07.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:07 smithi130 ceph-mon[21867]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:07.857 INFO:teuthology.orchestra.run.smithi150.stdout:Scheduled mon update... 2024-04-22T21:06:08.567 DEBUG:teuthology.orchestra.run.smithi130:mon.c> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.c.service 2024-04-22T21:06:08.570 DEBUG:teuthology.orchestra.run.smithi150:mon.b> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.b.service 2024-04-22T21:06:08.572 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-22T21:06:08.572 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T21:06:09.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:09.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi150:172.21.15.150=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:09.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: Saving service mon spec with placement smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi150:172.21.15.150=b;count:3 2024-04-22T21:06:09.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:09.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:09.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:09.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:09.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:06:09.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:06:09.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:09.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:09.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:06:09.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:09.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:08 smithi130 ceph-mon[21867]: Deploying daemon mon.c on smithi130 2024-04-22T21:06:10.258 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-22T21:06:10.258 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":1,"fsid":"dd7d46b8-00eb-11ef-bc93-c7b262605968","modified":"2024-04-22T21:04:51.246975Z","created":"2024-04-22T21:04:51.246975Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T21:06:10.260 INFO:teuthology.orchestra.run.smithi150.stderr:dumped monmap epoch 1 2024-04-22T21:06:11.208 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:10 smithi130 ceph-mon[21867]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:11.208 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:10 smithi130 ceph-mon[21867]: from='client.? 172.21.15.150:0/2333486464' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.208 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 podman[26375]: 2024-04-22 21:06:11.080729187 +0000 UTC m=+0.018272392 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T21:06:11.209 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 podman[26375]: 2024-04-22 21:06:11.213212843 +0000 UTC m=+0.150756058 container create 0e0f9de395abf1723872900f43ff8e152497183a38ba475235f4b4a1370c33c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, RELEASE=HEAD, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.buildah.version=1.19.8, distribution-scope=public, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.openshift.expose-services=, architecture=x86_64) 2024-04-22T21:06:11.468 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 podman[26375]: 2024-04-22 21:06:11.449042267 +0000 UTC m=+0.386585473 container init 0e0f9de395abf1723872900f43ff8e152497183a38ba475235f4b4a1370c33c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c, io.buildah.version=1.19.8, distribution-scope=public, GIT_BRANCH=HEAD, architecture=x86_64, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, build-date=2022-05-03T08:36:31.336870, release=754, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-22T21:06:11.468 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 podman[26375]: 2024-04-22 21:06:11.453100199 +0000 UTC m=+0.390643405 container start 0e0f9de395abf1723872900f43ff8e152497183a38ba475235f4b4a1370c33c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, maintainer=Guillaume Abrioux , release=754, io.openshift.expose-services=, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870) 2024-04-22T21:06:11.468 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T21:06:11.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-22T21:06:11.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: pidfile_write: ignore empty --pid-file 2024-04-22T21:06:11.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: load: jerasure load: lrc 2024-04-22T21:06:11.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: RocksDB version: 6.15.5 2024-04-22T21:06:11.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-22T21:06:11.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Compile date Apr 18 2022 2024-04-22T21:06:11.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: DB SUMMARY 2024-04-22T21:06:11.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: DB Session ID: WV0OGKXYXWYHVWKUDIJ3 2024-04-22T21:06:11.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: CURRENT file: CURRENT 2024-04-22T21:06:11.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T21:06:11.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-22T21:06:11.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-22T21:06:11.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.error_if_exists: 0 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.create_if_missing: 0 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.paranoid_checks: 1 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.env: 0x55b9facc6860 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.fs: Posix File System 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.info_log: 0x55b9fc9b3dc0 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.statistics: (nil) 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.use_fsync: 0 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_log_file_size: 0 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T21:06:11.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T21:06:11.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T21:06:11.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.allow_fallocate: 1 2024-04-22T21:06:11.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T21:06:11.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T21:06:11.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.use_direct_reads: 0 2024-04-22T21:06:11.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T21:06:11.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T21:06:11.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.db_log_dir: 2024-04-22T21:06:11.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-04-22T21:06:11.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T21:06:11.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T21:06:11.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T21:06:11.814 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T21:06:11.814 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T21:06:11.814 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T21:06:11.814 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T21:06:11.814 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T21:06:11.814 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.write_buffer_manager: 0x55b9fcaa4270 2024-04-22T21:06:11.814 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T21:06:11.814 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-22T21:06:11.814 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T21:06:11.814 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.unordered_write: 0 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.row_cache: None 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.wal_filter: None 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T21:06:11.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.preserve_deletes: 0 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.two_write_queues: 0 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.atomic_flush: 0 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.log_readahead_size: 0 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T21:06:11.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_background_jobs: 2 2024-04-22T21:06:11.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_background_compactions: -1 2024-04-22T21:06:11.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_subcompactions: 1 2024-04-22T21:06:11.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T21:06:11.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T21:06:11.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T21:06:11.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T21:06:11.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T21:06:11.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_open_files: -1 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_background_flushes: -1 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Compression algorithms supported: 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: kZSTD supported: 0 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: kXpressCompression supported: 0 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: kLZ4Compression supported: 1 2024-04-22T21:06:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: kBZip2Compression supported: 0 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: kZlibCompression supported: 1 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: kSnappyCompression supported: 1 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.merge_operator: 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compaction_filter: None 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b9fc97fd00) 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout: cache_index_and_filter_blocks: 1 2024-04-22T21:06:11.819 INFO:journalctl@ceph.mon.c.smithi130.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: pin_top_level_index_and_filter: 1 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_type: 0 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: data_block_index_type: 0 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_shortening: 1 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: hash_index_allow_collision: 1 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: checksum: 1 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: no_block_cache: 0 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache: 0x55b9fc9ea170 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_name: BinnedLRUCache 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_options: 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: capacity : 536870912 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: num_shard_bits : 4 2024-04-22T21:06:11.820 INFO:journalctl@ceph.mon.c.smithi130.stdout: strict_capacity_limit : 0 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: high_pri_pool_ratio: 0.000 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_compressed: (nil) 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: persistent_cache: (nil) 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_size: 4096 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_size_deviation: 10 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_restart_interval: 16 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_block_restart_interval: 1 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: metadata_block_size: 4096 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: partition_filters: 0 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_delta_encoding: 1 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: whole_key_filtering: 1 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: verify_compression: 0 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: read_amp_bytes_per_bit: 0 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: format_version: 4 2024-04-22T21:06:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout: enable_index_compression: 1 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_align: 0 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compression: NoCompression 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.num_levels: 7 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T21:06:11.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T21:06:11.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T21:06:11.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T21:06:11.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T21:06:11.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T21:06:11.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T21:06:11.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T21:06:11.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T21:06:11.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T21:06:11.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T21:06:11.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T21:06:11.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T21:06:11.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T21:06:11.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.arena_block_size: 4194304 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T21:06:11.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.table_properties_collectors: 2024-04-22T21:06:11.826 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.inplace_update_support: 0 2024-04-22T21:06:11.826 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T21:06:11.826 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T21:06:11.826 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T21:06:11.826 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T21:06:11.826 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.bloom_locality: 0 2024-04-22T21:06:11.826 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.max_successive_merges: 0 2024-04-22T21:06:11.826 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T21:06:11.826 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T21:06:11.826 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T21:06:11.826 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T21:06:11.826 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.ttl: 2592000 2024-04-22T21:06:11.826 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T21:06:11.827 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.enable_blob_files: false 2024-04-22T21:06:11.827 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.min_blob_size: 0 2024-04-22T21:06:11.827 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T21:06:11.827 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T21:06:11.827 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T21:06:11.827 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T21:06:11.827 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T21:06:11.827 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-22T21:06:11.827 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-22T21:06:11.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713819971478889, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-22T21:06:11.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-22T21:06:11.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-22T21:06:11.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713819971479187, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713819971, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "52a13a85-bf98-465b-bbf3-ff135a81440d", "db_session_id": "WV0OGKXYXWYHVWKUDIJ3"}} 2024-04-22T21:06:11.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-22T21:06:11.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713819971479485, "job": 1, "event": "recovery_finished"} 2024-04-22T21:06:11.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T21:06:11.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55b9fc9d0a80 2024-04-22T21:06:11.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: DB pointer 0x55b9fc9e0000 2024-04-22T21:06:11.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-22T21:06:11.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-22T21:06:11.828 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** DB Stats ** 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:11.829 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-22T21:06:11.830 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:06:11.831 INFO:journalctl@ceph.mon.c.smithi130.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: using public_addrv [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: starting mon.c rank -1 at public addrs [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] at bind addrs [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(???) e0 preinit fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(synchronizing).mds e1 new map 2024-04-22T21:06:11.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout: e1 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout: legacy client fscid: -1 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout: No filesystems configured 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T21:06:11.833 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T21:06:11.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mkfs dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:06:11.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T21:06:11.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T21:06:11.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: monmap e1: 1 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} 2024-04-22T21:06:11.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: fsmap 2024-04-22T21:06:11.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T21:06:11.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e1: no daemons active 2024-04-22T21:06:11.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1546795504' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:06:11.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/283215427' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T21:06:11.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/283215427' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-22T21:06:11.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/617492921' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:11.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T21:06:11.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: monmap e1: 1 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} 2024-04-22T21:06:11.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: fsmap 2024-04-22T21:06:11.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T21:06:11.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e1: no daemons active 2024-04-22T21:06:11.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3934197315' entity='client.admin' 2024-04-22T21:06:11.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2078182910' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T21:06:11.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1592184615' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T21:06:11.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Activating manager daemon y 2024-04-22T21:06:11.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e2: y(active, starting, since 0.00263134s) 2024-04-22T21:06:11.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:06:11.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:06:11.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:06:11.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:06:11.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Manager daemon y is now available 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14100 172.21.15.130:0/3882494552' entity='mgr.y' 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2034743182' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e3: y(active, since 1.00566s) 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e4: y(active, since 2s) 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3742667986' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3876330428' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3876330428' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1432866693' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-22T21:06:11.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1432866693' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-22T21:06:11.837 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e5: y(active, since 6s) 2024-04-22T21:06:11.837 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4080410037' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T21:06:11.837 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Active manager daemon y restarted 2024-04-22T21:06:11.837 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Activating manager daemon y 2024-04-22T21:06:11.837 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: osdmap e2: 0 total, 0 up, 0 in 2024-04-22T21:06:11.837 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e6: y(active, starting, since 0.462448s) 2024-04-22T21:06:11.838 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:06:11.838 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:06:11.838 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:06:11.838 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:06:11.838 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Manager daemon y is now available 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e7: y(active, since 1.46492s) 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: [22/Apr/2024:21:05:21] ENGINE Bus STARTING 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: [22/Apr/2024:21:05:21] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: [22/Apr/2024:21:05:21] ENGINE Bus STARTED 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:06:11.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e8: y(active, since 2s) 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Generating ssh key... 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi130", "addr": "172.21.15.130", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Deploying cephadm binary to smithi130 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Added host smithi130 2024-04-22T21:06:11.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:11.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Saving service mon spec with placement count:5 2024-04-22T21:06:11.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:06:11.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:11.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Saving service mgr spec with placement count:2 2024-04-22T21:06:11.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:06:11.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2420067331' entity='client.admin' 2024-04-22T21:06:11.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:06:11.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:06:11.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3732946745' entity='client.admin' 2024-04-22T21:06:11.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1155279535' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-22T21:06:11.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14120 172.21.15.130:0/2399506890' entity='mgr.y' 2024-04-22T21:06:11.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1155279535' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-22T21:06:11.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e9: y(active, since 16s) 2024-04-22T21:06:11.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3562640159' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T21:06:11.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Active manager daemon y restarted 2024-04-22T21:06:11.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Activating manager daemon y 2024-04-22T21:06:11.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: osdmap e3: 0 total, 0 up, 0 in 2024-04-22T21:06:11.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e10: y(active, starting, since 0.0524932s) 2024-04-22T21:06:11.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:06:11.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:06:11.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:06:11.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:06:11.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:06:11.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Manager daemon y is now available 2024-04-22T21:06:11.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:06:11.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:06:11.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: [22/Apr/2024:21:05:41] ENGINE Bus STARTING 2024-04-22T21:06:11.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: [22/Apr/2024:21:05:42] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:06:11.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: [22/Apr/2024:21:05:42] ENGINE Bus STARTED 2024-04-22T21:06:11.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-22T21:06:11.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e11: y(active, since 1.06035s) 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e12: y(active, since 2s) 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4199589592' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mgrmap e13: y(active, since 6s) 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3768518882' entity='client.admin' 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/714300523' entity='client.admin' 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi130", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi150", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:11.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Deploying cephadm binary to smithi150 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Added host smithi150 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi150", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:06:11.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3577576924' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3577576924' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi150:172.21.15.150=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Saving service mon spec with placement smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi150:172.21.15.150=b;count:3 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:06:11.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:11.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:06:11.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:11.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: Deploying daemon mon.c on smithi130 2024-04-22T21:06:11.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:11.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: from='client.? 172.21.15.150:0/2333486464' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T21:06:11.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 ceph-mon[26394]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-22T21:06:11.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 bash[26375]: 0e0f9de395abf1723872900f43ff8e152497183a38ba475235f4b4a1370c33c0 2024-04-22T21:06:11.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:11 smithi130 systemd[1]: Started Ceph mon.c for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:06:11.848 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-22T21:06:11.848 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T21:06:16.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:06:16.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: mon.a calling monitor election 2024-04-22T21:06:16.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:16.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:16.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:16.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:16.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: mon.c calling monitor election 2024-04-22T21:06:16.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:16.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:16.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:16.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:16.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-22T21:06:16.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: monmap e2: 2 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} 2024-04-22T21:06:16.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: fsmap 2024-04-22T21:06:16.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T21:06:16.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: mgrmap e13: y(active, since 34s) 2024-04-22T21:06:16.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: overall HEALTH_OK 2024-04-22T21:06:16.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:16.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:06:16.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:16.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:06:16.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: mon.a calling monitor election 2024-04-22T21:06:16.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:16.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:16.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: mon.c calling monitor election 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: monmap e2: 2 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: fsmap 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: mgrmap e13: y(active, since 34s) 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: overall HEALTH_OK 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:06:16.812 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:16 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:16.928 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-22T21:06:16.928 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":2,"fsid":"dd7d46b8-00eb-11ef-bc93-c7b262605968","modified":"2024-04-22T21:06:11.497275Z","created":"2024-04-22T21:04:51.246975Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3301","nonce":0},{"type":"v1","addr":"172.21.15.130:6790","nonce":0}]},"addr":"172.21.15.130:6790/0","public_addr":"172.21.15.130:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-22T21:06:16.930 INFO:teuthology.orchestra.run.smithi150.stderr:dumped monmap epoch 2 2024-04-22T21:06:17.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:17 smithi130 ceph-mon[21867]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:17.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:17 smithi130 ceph-mon[21867]: Deploying daemon mon.b on smithi150 2024-04-22T21:06:17.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:17 smithi130 ceph-mon[21867]: from='client.? 172.21.15.150:0/2439336803' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T21:06:17.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:17 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:17.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:17 smithi130 ceph-mon[26394]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:17.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:17 smithi130 ceph-mon[26394]: Deploying daemon mon.b on smithi150 2024-04-22T21:06:17.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:17 smithi130 ceph-mon[26394]: from='client.? 172.21.15.150:0/2439336803' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T21:06:17.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:17 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:19.273 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-22T21:06:19.273 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T21:06:19.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:19 smithi130 ceph-mon[26394]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:19.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:19 smithi130 ceph-mon[21867]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:20.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:20 smithi150 systemd[1]: Starting Ceph mon.b for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:06:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-22T21:06:21.442 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":2,"fsid":"dd7d46b8-00eb-11ef-bc93-c7b262605968","modified":"2024-04-22T21:06:11.497275Z","created":"2024-04-22T21:04:51.246975Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3301","nonce":0},{"type":"v1","addr":"172.21.15.130:6790","nonce":0}]},"addr":"172.21.15.130:6790/0","public_addr":"172.21.15.130:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-22T21:06:21.444 INFO:teuthology.orchestra.run.smithi150.stderr:dumped monmap epoch 2 2024-04-22T21:06:21.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:21 smithi150 podman[22642]: 2024-04-22 21:06:21.454356516 +0000 UTC m=+0.107267175 container create 861909a68cf92d263def712ac54b5359bddb0a9aad280c48be1096b13c849172 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b, build-date=2022-05-03T08:36:31.336870, release=754, io.buildah.version=1.19.8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, ceph=True, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0) 2024-04-22T21:06:21.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:21 smithi150 podman[22642]: 2024-04-22 21:06:21.365652626 +0000 UTC m=+0.018563294 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T21:06:22.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:21 smithi130 ceph-mon[21867]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:22.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:21 smithi130 ceph-mon[21867]: from='client.? 172.21.15.150:0/1428778990' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T21:06:22.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:21 smithi130 ceph-mon[26394]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:22.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:21 smithi130 ceph-mon[26394]: from='client.? 172.21.15.150:0/1428778990' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T21:06:22.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 podman[22642]: 2024-04-22 21:06:22.201511969 +0000 UTC m=+0.854422626 container init 861909a68cf92d263def712ac54b5359bddb0a9aad280c48be1096b13c849172 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b, io.openshift.tags=base centos centos-stream, release=754, GIT_CLEAN=True, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, ceph=True, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.buildah.version=1.19.8, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0) 2024-04-22T21:06:22.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 podman[22642]: 2024-04-22 21:06:22.205749117 +0000 UTC m=+0.858659783 container start 861909a68cf92d263def712ac54b5359bddb0a9aad280c48be1096b13c849172 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b, name=centos-stream, release=754, ceph=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, architecture=x86_64, io.openshift.expose-services=) 2024-04-22T21:06:22.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T21:06:22.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-22T21:06:22.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: pidfile_write: ignore empty --pid-file 2024-04-22T21:06:22.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: load: jerasure load: lrc 2024-04-22T21:06:22.228 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: RocksDB version: 6.15.5 2024-04-22T21:06:22.228 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-22T21:06:22.228 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Compile date Apr 18 2022 2024-04-22T21:06:22.228 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: DB SUMMARY 2024-04-22T21:06:22.228 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: DB Session ID: LOLW9QUORLC0QYDKDFZA 2024-04-22T21:06:22.228 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: CURRENT file: CURRENT 2024-04-22T21:06:22.228 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T21:06:22.228 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-22T21:06:22.228 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-04-22T21:06:22.228 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-04-22T21:06:22.228 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.error_if_exists: 0 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.create_if_missing: 0 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.paranoid_checks: 1 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.env: 0x55f9468c6860 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.fs: Posix File System 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.info_log: 0x55f947297dc0 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.statistics: (nil) 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.use_fsync: 0 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_log_file_size: 0 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T21:06:22.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T21:06:22.230 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.allow_fallocate: 1 2024-04-22T21:06:22.230 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T21:06:22.230 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T21:06:22.230 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.use_direct_reads: 0 2024-04-22T21:06:22.230 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T21:06:22.230 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T21:06:22.230 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.db_log_dir: 2024-04-22T21:06:22.230 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-04-22T21:06:22.230 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T21:06:22.230 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T21:06:22.230 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T21:06:22.230 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T21:06:22.231 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T21:06:22.231 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T21:06:22.232 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T21:06:22.232 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T21:06:22.232 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.write_buffer_manager: 0x55f947388240 2024-04-22T21:06:22.232 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T21:06:22.232 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-22T21:06:22.232 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T21:06:22.232 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T21:06:22.232 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T21:06:22.232 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T21:06:22.232 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.unordered_write: 0 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.row_cache: None 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.wal_filter: None 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.preserve_deletes: 0 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.two_write_queues: 0 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T21:06:22.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.atomic_flush: 0 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.log_readahead_size: 0 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_background_jobs: 2 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_background_compactions: -1 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_subcompactions: 1 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T21:06:22.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_open_files: -1 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_background_flushes: -1 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Compression algorithms supported: 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: kZSTD supported: 0 2024-04-22T21:06:22.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: kXpressCompression supported: 0 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: kLZ4Compression supported: 1 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: kBZip2Compression supported: 0 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: kZlibCompression supported: 1 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: kSnappyCompression supported: 1 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.merge_operator: 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compaction_filter: None 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T21:06:22.236 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f947263d00) 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: cache_index_and_filter_blocks: 1 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: pin_top_level_index_and_filter: 1 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: index_type: 0 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: data_block_index_type: 0 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: index_shortening: 1 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: hash_index_allow_collision: 1 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: checksum: 1 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: no_block_cache: 0 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache: 0x55f9472ce170 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache_name: BinnedLRUCache 2024-04-22T21:06:22.237 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache_options: 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: capacity : 536870912 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: num_shard_bits : 4 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: strict_capacity_limit : 0 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: high_pri_pool_ratio: 0.000 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache_compressed: (nil) 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: persistent_cache: (nil) 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_size: 4096 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_size_deviation: 10 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_restart_interval: 16 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: index_block_restart_interval: 1 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: metadata_block_size: 4096 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: partition_filters: 0 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_delta_encoding: 1 2024-04-22T21:06:22.238 INFO:journalctl@ceph.mon.b.smithi150.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout: whole_key_filtering: 1 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout: verify_compression: 0 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout: read_amp_bytes_per_bit: 0 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout: format_version: 4 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout: enable_index_compression: 1 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_align: 0 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compression: NoCompression 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.num_levels: 7 2024-04-22T21:06:22.239 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T21:06:22.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T21:06:22.241 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.arena_block_size: 4194304 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T21:06:22.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.table_properties_collectors: 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.inplace_update_support: 0 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.bloom_locality: 0 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.max_successive_merges: 0 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T21:06:22.243 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.ttl: 2592000 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.enable_blob_files: false 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.min_blob_size: 0 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713819982232391, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-22T21:06:22.244 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713819982232800, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713819982, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "0993097f-d96d-4563-82b8-d8c336ee647f", "db_session_id": "LOLW9QUORLC0QYDKDFZA"}} 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713819982233229, "job": 1, "event": "recovery_finished"} 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55f9472b4a80 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: DB pointer 0x55f9472c4000 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** DB Stats ** 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T21:06:22.245 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:22.246 INFO:journalctl@ceph.mon.b.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative compaction: 0.00 GB write, 0.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval compaction: 0.00 GB write, 0.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:06:22.247 INFO:journalctl@ceph.mon.b.smithi150.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.9 0.00 0.00 1 0.000 0 0 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative compaction: 0.00 GB write, 0.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:06:22.248 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:06:22.249 INFO:journalctl@ceph.mon.b.smithi150.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T21:06:22.249 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2024-04-22T21:06:22.249 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T21:06:22.249 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-22T21:06:22.249 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: using public_addr v2:172.21.15.150:0/0 -> [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] 2024-04-22T21:06:22.249 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: starting mon.b rank -1 at public addrs [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] at bind addrs [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:06:22.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(???) e0 preinit fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:06:22.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing).mds e1 new map 2024-04-22T21:06:22.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-22T21:06:22.613 INFO:journalctl@ceph.mon.b.smithi150.stdout: e1 2024-04-22T21:06:22.613 INFO:journalctl@ceph.mon.b.smithi150.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T21:06:22.613 INFO:journalctl@ceph.mon.b.smithi150.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T21:06:22.613 INFO:journalctl@ceph.mon.b.smithi150.stdout: legacy client fscid: -1 2024-04-22T21:06:22.614 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2024-04-22T21:06:22.614 INFO:journalctl@ceph.mon.b.smithi150.stdout: No filesystems configured 2024-04-22T21:06:22.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-22T21:06:22.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-22T21:06:22.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-22T21:06:22.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-22T21:06:22.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-22T21:06:22.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-22T21:06:22.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-22T21:06:22.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T21:06:22.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T21:06:22.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T21:06:22.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:22.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: Deploying daemon mon.b on smithi150 2024-04-22T21:06:22.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: from='client.? 172.21.15.150:0/2439336803' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T21:06:22.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:22.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:22.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:22.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: from='client.? 172.21.15.150:0/1428778990' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T21:06:22.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing).paxosservice(auth 1..4) refresh upgraded, format 0 -> 3 2024-04-22T21:06:22.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 ceph-mon[22676]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-22T21:06:22.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 bash[22642]: 861909a68cf92d263def712ac54b5359bddb0a9aad280c48be1096b13c849172 2024-04-22T21:06:22.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:22 smithi150 systemd[1]: Started Ceph mon.b for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:06:23.110 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-22T21:06:23.111 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T21:06:27.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:27.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:06:27.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: mon.a calling monitor election 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: mon.c calling monitor election 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: monmap e3: 3 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: fsmap 2024-04-22T21:06:27.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T21:06:27.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: mgrmap e13: y(active, since 45s) 2024-04-22T21:06:27.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-22T21:06:27.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-22T21:06:27.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-22T21:06:27.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: mon.b (rank 2) addr [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] is down (out of quorum) 2024-04-22T21:06:27.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:27.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:27.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:27.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:27.561 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:27.561 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:06:27.562 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: mon.a calling monitor election 2024-04-22T21:06:27.562 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: mon.c calling monitor election 2024-04-22T21:06:27.562 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:27.562 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:27.562 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:27.562 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:27.562 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:27.562 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:27.562 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:27.562 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:27.562 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:27.562 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-22T21:06:27.562 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: monmap e3: 3 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} 2024-04-22T21:06:27.563 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: fsmap 2024-04-22T21:06:27.563 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T21:06:27.563 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: mgrmap e13: y(active, since 45s) 2024-04-22T21:06:27.563 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-22T21:06:27.563 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-22T21:06:27.563 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-22T21:06:27.563 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: mon.b (rank 2) addr [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] is down (out of quorum) 2024-04-22T21:06:27.563 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:27.563 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:27.563 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:27.563 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:27.998 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-22T21:06:27.998 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":3,"fsid":"dd7d46b8-00eb-11ef-bc93-c7b262605968","modified":"2024-04-22T21:06:22.254615Z","created":"2024-04-22T21:04:51.246975Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3301","nonce":0},{"type":"v1","addr":"172.21.15.130:6790","nonce":0}]},"addr":"172.21.15.130:6790/0","public_addr":"172.21.15.130:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:3300","nonce":0},{"type":"v1","addr":"172.21.15.150:6789","nonce":0}]},"addr":"172.21.15.150:6789/0","public_addr":"172.21.15.150:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-22T21:06:28.002 INFO:teuthology.orchestra.run.smithi150.stderr:dumped monmap epoch 3 2024-04-22T21:06:28.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:28 smithi130 ceph-mon[21867]: from='client.? 172.21.15.150:0/1803150908' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T21:06:28.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:28 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:28.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:28 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:28.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:28 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:28.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:28 smithi130 ceph-mon[26394]: from='client.? 172.21.15.150:0/1803150908' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T21:06:28.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:28 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:28.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:28 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:28.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:28 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:28.820 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-22T21:06:28.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-22T21:06:29.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:29.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:06:29.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: mon.a calling monitor election 2024-04-22T21:06:29.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: mon.c calling monitor election 2024-04-22T21:06:29.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:29.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:06:29.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:29.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:29.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:29.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:29.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:29.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:29.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:29.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-22T21:06:29.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: monmap e3: 3 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} 2024-04-22T21:06:29.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: fsmap 2024-04-22T21:06:29.614 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T21:06:29.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: mgrmap e13: y(active, since 45s) 2024-04-22T21:06:29.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-22T21:06:29.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-22T21:06:29.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-22T21:06:29.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: mon.b (rank 2) addr [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] is down (out of quorum) 2024-04-22T21:06:29.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:29.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:29.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:29.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:29.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='client.? 172.21.15.150:0/1803150908' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T21:06:29.615 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:29.616 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:29.616 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:30.125 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: mon.b calling monitor election 2024-04-22T21:06:30.125 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: mon.b calling monitor election 2024-04-22T21:06:30.125 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: mon.c calling monitor election 2024-04-22T21:06:30.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: mon.a calling monitor election 2024-04-22T21:06:30.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T21:06:30.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: monmap e3: 3 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} 2024-04-22T21:06:30.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: fsmap 2024-04-22T21:06:30.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T21:06:30.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: mgrmap e13: y(active, since 47s) 2024-04-22T21:06:30.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-22T21:06:30.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: Cluster is now healthy 2024-04-22T21:06:30.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: overall HEALTH_OK 2024-04-22T21:06:30.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:30.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:06:30.127 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:06:30.127 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:30.127 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: mon.b calling monitor election 2024-04-22T21:06:30.127 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: mon.b calling monitor election 2024-04-22T21:06:30.127 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: mon.c calling monitor election 2024-04-22T21:06:30.127 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: mon.a calling monitor election 2024-04-22T21:06:30.127 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T21:06:30.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: monmap e3: 3 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} 2024-04-22T21:06:30.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: fsmap 2024-04-22T21:06:30.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T21:06:30.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: mgrmap e13: y(active, since 47s) 2024-04-22T21:06:30.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-22T21:06:30.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: Cluster is now healthy 2024-04-22T21:06:30.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: overall HEALTH_OK 2024-04-22T21:06:30.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:30.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:06:30.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:06:30.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:29 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:30.196 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: mon.b calling monitor election 2024-04-22T21:06:30.196 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: mon.b calling monitor election 2024-04-22T21:06:30.196 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: mon.c calling monitor election 2024-04-22T21:06:30.196 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: mon.a calling monitor election 2024-04-22T21:06:30.196 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T21:06:30.196 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: monmap e3: 3 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} 2024-04-22T21:06:30.196 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: fsmap 2024-04-22T21:06:30.196 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T21:06:30.196 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: mgrmap e13: y(active, since 47s) 2024-04-22T21:06:30.197 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-22T21:06:30.197 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: Cluster is now healthy 2024-04-22T21:06:30.197 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: overall HEALTH_OK 2024-04-22T21:06:30.197 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:30.197 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:06:30.197 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:06:30.197 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:29 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:30.979 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:30 smithi130 ceph-mon[21867]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:30.979 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:30 smithi130 ceph-mon[21867]: Reconfiguring mon.c (monmap changed)... 2024-04-22T21:06:30.979 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:30 smithi130 ceph-mon[21867]: Reconfiguring daemon mon.c on smithi130 2024-04-22T21:06:30.979 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:30 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:30.979 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:30 smithi130 ceph-mon[26394]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:30.979 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:30 smithi130 ceph-mon[26394]: Reconfiguring mon.c (monmap changed)... 2024-04-22T21:06:30.979 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:30 smithi130 ceph-mon[26394]: Reconfiguring daemon mon.c on smithi130 2024-04-22T21:06:30.979 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:30 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:31.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:30 smithi150 ceph-mon[22676]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:31.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:30 smithi150 ceph-mon[22676]: Reconfiguring mon.c (monmap changed)... 2024-04-22T21:06:31.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:30 smithi150 ceph-mon[22676]: Reconfiguring daemon mon.c on smithi130 2024-04-22T21:06:31.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:30 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:06:32.689 INFO:teuthology.orchestra.run.smithi130.stdout:# minimal ceph.conf for dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:06:32.689 INFO:teuthology.orchestra.run.smithi130.stdout:[global] 2024-04-22T21:06:32.689 INFO:teuthology.orchestra.run.smithi130.stdout: fsid = dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:06:32.689 INFO:teuthology.orchestra.run.smithi130.stdout: mon_host = [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] 2024-04-22T21:06:32.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[21867]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:32.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:32.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:32.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[21867]: Reconfiguring mon.b (monmap changed)... 2024-04-22T21:06:32.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:06:32.969 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:06:32.969 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:32.969 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[21867]: Reconfiguring daemon mon.b on smithi150 2024-04-22T21:06:32.969 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/477565653' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:32.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[26394]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:32.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:32.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:32.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[26394]: Reconfiguring mon.b (monmap changed)... 2024-04-22T21:06:32.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:06:32.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:06:32.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:32.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[26394]: Reconfiguring daemon mon.b on smithi150 2024-04-22T21:06:32.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:32 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/477565653' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:33.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:32 smithi150 ceph-mon[22676]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:33.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:32 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:33.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:32 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:33.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:32 smithi150 ceph-mon[22676]: Reconfiguring mon.b (monmap changed)... 2024-04-22T21:06:33.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:32 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:06:33.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:32 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:06:33.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:32 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:33.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:32 smithi150 ceph-mon[22676]: Reconfiguring daemon mon.b on smithi150 2024-04-22T21:06:33.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:32 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/477565653' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:33.312 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-22T21:06:33.312 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-22T21:06:33.312 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.conf 2024-04-22T21:06:33.337 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-22T21:06:33.337 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:06:33.401 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-22T21:06:33.401 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/ceph/ceph.conf 2024-04-22T21:06:33.427 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-22T21:06:33.427 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:06:33.491 INFO:tasks.cephadm:Adding mgr.y on smithi130 2024-04-22T21:06:33.491 INFO:tasks.cephadm:Adding mgr.x on smithi150 2024-04-22T21:06:33.492 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch apply mgr '2;smithi130=y;smithi150=x' 2024-04-22T21:06:34.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:34 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:34.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:34 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:34.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:34 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:34.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:34 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:34.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:34 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:34.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:34 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:34.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:34 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:34.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:34 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:34.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:34 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:34.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:34 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:34.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:34 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:34.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:34 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:34.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:34 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:34.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:34 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:34.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:34 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:35.253 INFO:teuthology.orchestra.run.smithi150.stdout:Scheduled mgr update... 2024-04-22T21:06:35.549 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:35 smithi150 ceph-mon[22676]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:35.549 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:35 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:35.549 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:35 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:35.549 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:35 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:35.549 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:35 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:35.549 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:35 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:35.550 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:35 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:06:35.550 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:35 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-22T21:06:35.550 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:35 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:06:35.550 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:35 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[21867]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:06:35.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-22T21:06:35.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:06:35.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[26394]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:06:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-22T21:06:35.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:06:35.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:35 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:35.821 DEBUG:teuthology.orchestra.run.smithi150:mgr.x> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.x.service 2024-04-22T21:06:35.823 INFO:tasks.cephadm:Deploying OSDs... 2024-04-22T21:06:35.824 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-22T21:06:35.824 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/scratch_devs of=/dev/stdout 2024-04-22T21:06:35.845 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-22T21:06:35.845 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_1 2024-04-22T21:06:35.904 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-22T21:06:35.904 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T21:06:35.904 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-04-22T21:06:35.904 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T21:06:35.904 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T21:06:35.905 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-04-22 21:06:10.902033427 +0000 2024-04-22T21:06:35.905 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-04-22 21:03:51.260755191 +0000 2024-04-22T21:06:35.905 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-04-22 21:03:51.260755191 +0000 2024-04-22T21:06:35.905 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-04-22 21:03:51.260755191 +0000 2024-04-22T21:06:35.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-22T21:06:35.972 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-04-22T21:06:35.972 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-04-22T21:06:35.972 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000208263 s, 2.5 MB/s 2024-04-22T21:06:35.973 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-22T21:06:36.031 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_2 2024-04-22T21:06:36.088 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-22T21:06:36.088 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T21:06:36.089 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 683 Links: 1 2024-04-22T21:06:36.089 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T21:06:36.089 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T21:06:36.089 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-04-22 21:06:10.902033427 +0000 2024-04-22T21:06:36.089 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-04-22 21:03:51.584748935 +0000 2024-04-22T21:06:36.089 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-04-22 21:03:51.584748935 +0000 2024-04-22T21:06:36.089 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-04-22 21:03:51.584748935 +0000 2024-04-22T21:06:36.089 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-22T21:06:36.153 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-04-22T21:06:36.153 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-04-22T21:06:36.153 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000211743 s, 2.4 MB/s 2024-04-22T21:06:36.154 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-22T21:06:36.211 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_3 2024-04-22T21:06:36.267 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-22T21:06:36.267 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T21:06:36.267 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-22T21:06:36.267 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T21:06:36.268 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T21:06:36.268 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-04-22 21:06:10.903033408 +0000 2024-04-22T21:06:36.268 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-04-22 21:03:51.917742505 +0000 2024-04-22T21:06:36.268 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-04-22 21:03:51.917742505 +0000 2024-04-22T21:06:36.268 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-04-22 21:03:51.917742505 +0000 2024-04-22T21:06:36.268 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-22T21:06:36.330 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-04-22T21:06:36.331 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-04-22T21:06:36.331 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000250167 s, 2.0 MB/s 2024-04-22T21:06:36.332 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-22T21:06:36.389 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_4 2024-04-22T21:06:36.445 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-22T21:06:36.445 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T21:06:36.445 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-04-22T21:06:36.445 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T21:06:36.445 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T21:06:36.446 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-04-22 21:06:10.903033408 +0000 2024-04-22T21:06:36.446 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-04-22 21:03:52.256735959 +0000 2024-04-22T21:06:36.446 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-04-22 21:03:52.256735959 +0000 2024-04-22T21:06:36.446 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-04-22 21:03:52.256735959 +0000 2024-04-22T21:06:36.446 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-22T21:06:36.508 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-04-22T21:06:36.508 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-04-22T21:06:36.508 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000204957 s, 2.5 MB/s 2024-04-22T21:06:36.509 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-22T21:06:36.565 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-22T21:06:36.565 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/scratch_devs of=/dev/stdout 2024-04-22T21:06:36.581 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-22T21:06:36.581 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_1 2024-04-22T21:06:36.636 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-22T21:06:36.636 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T21:06:36.636 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-04-22T21:06:36.637 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T21:06:36.637 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T21:06:36.637 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2024-04-22 21:06:20.328733671 +0000 2024-04-22T21:06:36.637 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2024-04-22 21:03:51.383595669 +0000 2024-04-22T21:06:36.637 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2024-04-22 21:03:51.383595669 +0000 2024-04-22T21:06:36.637 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: 2024-04-22 21:03:51.383595669 +0000 2024-04-22T21:06:36.637 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-22T21:06:36.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:36 smithi150 ceph-mon[22676]: from='client.34109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi130=y;smithi150=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:36.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:36 smithi150 ceph-mon[22676]: Saving service mgr spec with placement smithi130=y;smithi150=x;count:2 2024-04-22T21:06:36.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:36 smithi150 ceph-mon[22676]: Deploying daemon mgr.x on smithi150 2024-04-22T21:06:36.686 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2024-04-22T21:06:36.686 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2024-04-22T21:06:36.686 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000246905 s, 2.1 MB/s 2024-04-22T21:06:36.687 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-22T21:06:36.743 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_2 2024-04-22T21:06:36.798 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-22T21:06:36.798 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T21:06:36.798 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-04-22T21:06:36.798 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T21:06:36.798 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T21:06:36.798 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2024-04-22 21:06:20.328733671 +0000 2024-04-22T21:06:36.799 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2024-04-22 21:03:51.732588744 +0000 2024-04-22T21:06:36.799 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2024-04-22 21:03:51.732588744 +0000 2024-04-22T21:06:36.799 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: 2024-04-22 21:03:51.732588744 +0000 2024-04-22T21:06:36.799 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-22T21:06:36.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:36 smithi130 ceph-mon[21867]: from='client.34109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi130=y;smithi150=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:36.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:36 smithi130 ceph-mon[21867]: Saving service mgr spec with placement smithi130=y;smithi150=x;count:2 2024-04-22T21:06:36.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:36 smithi130 ceph-mon[21867]: Deploying daemon mgr.x on smithi150 2024-04-22T21:06:36.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:36 smithi130 ceph-mon[26394]: from='client.34109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi130=y;smithi150=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:36.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:36 smithi130 ceph-mon[26394]: Saving service mgr spec with placement smithi130=y;smithi150=x;count:2 2024-04-22T21:06:36.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:36 smithi130 ceph-mon[26394]: Deploying daemon mgr.x on smithi150 2024-04-22T21:06:36.861 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2024-04-22T21:06:36.861 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2024-04-22T21:06:36.861 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000212763 s, 2.4 MB/s 2024-04-22T21:06:36.862 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-22T21:06:36.918 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_3 2024-04-22T21:06:36.972 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-22T21:06:36.973 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T21:06:36.973 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-04-22T21:06:36.973 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T21:06:36.973 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T21:06:36.973 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2024-04-22 21:06:20.329733652 +0000 2024-04-22T21:06:36.973 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2024-04-22 21:03:52.071582017 +0000 2024-04-22T21:06:36.973 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2024-04-22 21:03:52.071582017 +0000 2024-04-22T21:06:36.973 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: 2024-04-22 21:03:52.071582017 +0000 2024-04-22T21:06:36.974 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-22T21:06:37.089 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2024-04-22T21:06:37.090 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2024-04-22T21:06:37.090 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000201575 s, 2.5 MB/s 2024-04-22T21:06:37.091 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-22T21:06:37.106 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_4 2024-04-22T21:06:37.165 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-22T21:06:37.165 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T21:06:37.165 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-22T21:06:37.165 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T21:06:37.165 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T21:06:37.165 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2024-04-22 21:06:37.053416501 +0000 2024-04-22T21:06:37.165 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2024-04-22 21:03:52.418575132 +0000 2024-04-22T21:06:37.165 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2024-04-22 21:03:52.418575132 +0000 2024-04-22T21:06:37.166 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: 2024-04-22 21:03:52.418575132 +0000 2024-04-22T21:06:37.166 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-22T21:06:37.265 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2024-04-22T21:06:37.265 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2024-04-22T21:06:37.265 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000187558 s, 2.7 MB/s 2024-04-22T21:06:37.267 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-22T21:06:37.282 INFO:tasks.cephadm:Deploying osd.0 on smithi130 with /dev/vg_nvme/lv_4... 2024-04-22T21:06:37.283 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-22T21:06:37.503 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:37 smithi150 ceph-mon[22676]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:37.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:37 smithi130 ceph-mon[21867]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:37.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:37 smithi130 ceph-mon[26394]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:37.784 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:37 smithi150 ceph-mgr[23856]: -- 172.21.15.150:0/3454441254 <== mon.2 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558ef8e90340 con 0x558ef8e68c00 2024-04-22T21:06:38.070 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:37 smithi150 bash[23837]: 2c373781789782e87c284a96c7c4c2dfd8ba357f1b3d942146247bdb69eb5055 2024-04-22T21:06:38.070 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:37 smithi150 systemd[1]: Started Ceph mgr.x for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:06:38.071 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:37 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:37.834+0000 7f1962c1f000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:06:38.071 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:37 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:37.971+0000 7f1962c1f000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:06:38.071 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:38.074+0000 7f1962c1f000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:06:38.806 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:38.397+0000 7f1962c1f000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:06:38.806 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:38.482+0000 7f1962c1f000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:06:39.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:38 smithi150 ceph-mon[22676]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:39.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:39.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:39.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:39.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:39.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:39.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:39.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:06:39.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:06:39.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:39.201 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[21867]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:39.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:39.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:39.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:39.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:39.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:39.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:39.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:06:39.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:06:39.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:39.202 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[26394]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:39.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:39.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:39.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:39.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:39.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:39.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:39.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:06:39.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:06:39.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:39.357 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-22T21:06:39.358 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-22T21:06:39.358 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2024-04-22T21:06:39.358 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2024-04-22T21:06:39.358 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111674 s, 939 MB/s 2024-04-22T21:06:39.358 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2024-04-22T21:06:39.937 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:39.627+0000 7f1962c1f000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:06:39.937 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:39.787+0000 7f1962c1f000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:06:39.937 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:39.867+0000 7f1962c1f000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:06:40.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:39 smithi130 ceph-mon[21867]: Reconfiguring mgr.y (unknown last config time)... 2024-04-22T21:06:40.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:39 smithi130 ceph-mon[21867]: Reconfiguring daemon mgr.y on smithi130 2024-04-22T21:06:40.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:39 smithi130 ceph-mon[26394]: Reconfiguring mgr.y (unknown last config time)... 2024-04-22T21:06:40.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:39 smithi130 ceph-mon[26394]: Reconfiguring daemon mgr.y on smithi130 2024-04-22T21:06:40.263 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:39 smithi150 ceph-mon[22676]: Reconfiguring mgr.y (unknown last config time)... 2024-04-22T21:06:40.264 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:39 smithi150 ceph-mon[22676]: Reconfiguring daemon mgr.y on smithi130 2024-04-22T21:06:40.264 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:40 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:40.178+0000 7f1962c1f000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:06:40.264 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:40 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:40.268+0000 7f1962c1f000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:06:40.596 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:40 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:40.360+0000 7f1962c1f000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:06:40.596 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:40 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:40.600+0000 7f1962c1f000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:06:40.848 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_4 2024-04-22T21:06:41.363 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:41 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:41.036+0000 7f1962c1f000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:06:41.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:41 smithi150 ceph-mon[22676]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:41.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:41 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:41.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:41 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:41.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:41 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:41.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:41 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:41.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:41 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:41.469 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:41 smithi130 ceph-mon[21867]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:41.469 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:41 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:41.469 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:41 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:41.469 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:41 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:41.469 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:41 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:41.469 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:41 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:41.470 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:41 smithi130 ceph-mon[26394]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:41.470 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:41 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:41.470 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:41 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:41.470 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:41 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:41.470 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:41 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:41.470 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:41 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:41.863 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:41 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:41.522+0000 7f1962c1f000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:06:41.863 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:41 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:41.614+0000 7f1962c1f000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:06:42.200 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:41 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:41.905+0000 7f1962c1f000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:06:42.201 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:41 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:41.993+0000 7f1962c1f000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:06:42.407 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:06:42.407 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:06:42.407 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:42.407 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:42 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:06:42.407 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:42 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:06:42.407 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:42 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:42.529 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:42 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:06:42.529 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:42 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:06:42.529 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:42 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:42.530 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:42 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:42.402+0000 7f1962c1f000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:06:42.530 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:42 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:42.534+0000 7f1962c1f000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:06:42.820 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:42 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:42.625+0000 7f1962c1f000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:06:42.820 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:42 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:42.738+0000 7f1962c1f000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:06:42.820 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:42 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:42.825+0000 7f1962c1f000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:06:43.399 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:43 smithi130 ceph-mon[21867]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:43.399 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:43 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:06:43.399 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:43 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:06:43.399 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:43 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:43.399 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:43 smithi130 ceph-mon[26394]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:43.399 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:43 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:06:43.400 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:43 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:06:43.400 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:43 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:43.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:43 smithi150 ceph-mon[22676]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:43.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:43 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:06:43.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:43 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:06:43.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:43 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:43.613 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:43.367+0000 7f1962c1f000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:06:44.112 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:06:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:06:43.756+0000 7f1962c1f000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:06:44.454 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:44 smithi130 ceph-mon[26394]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:44.454 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:44 smithi130 ceph-mon[26394]: Standby manager daemon x started 2024-04-22T21:06:44.454 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:44 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/1428550679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:06:44.454 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:44 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/1428550679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:06:44.454 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:44 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/1428550679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:06:44.454 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:44 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/1428550679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:06:44.455 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:44 smithi130 ceph-mon[21867]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:44.455 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:44 smithi130 ceph-mon[21867]: Standby manager daemon x started 2024-04-22T21:06:44.455 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:44 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/1428550679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:06:44.455 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:44 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/1428550679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:06:44.455 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:44 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/1428550679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:06:44.455 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:44 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/1428550679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:06:44.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:44 smithi150 ceph-mon[22676]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:06:44.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:44 smithi150 ceph-mon[22676]: Standby manager daemon x started 2024-04-22T21:06:44.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:44 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/1428550679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:06:44.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:44 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/1428550679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:06:44.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:44 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/1428550679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:06:44.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:44 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/1428550679' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:06:45.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[21867]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:45.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[21867]: mgrmap e14: y(active, since 62s), standbys: x 2024-04-22T21:06:45.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:06:45.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2760509076' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8e1579b-9111-4278-8801-1892b56c913a"}]: dispatch 2024-04-22T21:06:45.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8e1579b-9111-4278-8801-1892b56c913a"}]: dispatch 2024-04-22T21:06:45.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b8e1579b-9111-4278-8801-1892b56c913a"}]': finished 2024-04-22T21:06:45.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[21867]: osdmap e5: 1 total, 0 up, 1 in 2024-04-22T21:06:45.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:06:45.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[26394]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:45.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[26394]: mgrmap e14: y(active, since 62s), standbys: x 2024-04-22T21:06:45.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:06:45.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2760509076' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8e1579b-9111-4278-8801-1892b56c913a"}]: dispatch 2024-04-22T21:06:45.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[26394]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8e1579b-9111-4278-8801-1892b56c913a"}]: dispatch 2024-04-22T21:06:45.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[26394]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b8e1579b-9111-4278-8801-1892b56c913a"}]': finished 2024-04-22T21:06:45.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[26394]: osdmap e5: 1 total, 0 up, 1 in 2024-04-22T21:06:45.561 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:45 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:06:45.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:45 smithi150 ceph-mon[22676]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:45.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:45 smithi150 ceph-mon[22676]: mgrmap e14: y(active, since 62s), standbys: x 2024-04-22T21:06:45.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:45 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:06:45.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:45 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2760509076' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8e1579b-9111-4278-8801-1892b56c913a"}]: dispatch 2024-04-22T21:06:45.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:45 smithi150 ceph-mon[22676]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8e1579b-9111-4278-8801-1892b56c913a"}]: dispatch 2024-04-22T21:06:45.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:45 smithi150 ceph-mon[22676]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b8e1579b-9111-4278-8801-1892b56c913a"}]': finished 2024-04-22T21:06:45.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:45 smithi150 ceph-mon[22676]: osdmap e5: 1 total, 0 up, 1 in 2024-04-22T21:06:45.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:45 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:06:46.477 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:46 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/608707334' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:06:46.477 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:46 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/608707334' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:06:46.514 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:46 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/608707334' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:06:47.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:47 smithi130 ceph-mon[21867]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:47.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:47 smithi130 ceph-mon[26394]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:47.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:47 smithi150 ceph-mon[22676]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:49.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:49 smithi130 ceph-mon[21867]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:49.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:49 smithi130 ceph-mon[26394]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:49.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:49 smithi150 ceph-mon[22676]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:51.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:51 smithi130 ceph-mon[21867]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:51.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:51 smithi130 ceph-mon[26394]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:51.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:51 smithi150 ceph-mon[22676]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:52.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T21:06:52.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:52.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:52 smithi130 ceph-mon[21867]: Deploying daemon osd.0 on smithi130 2024-04-22T21:06:52.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T21:06:52.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:52.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:52 smithi130 ceph-mon[26394]: Deploying daemon osd.0 on smithi130 2024-04-22T21:06:52.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T21:06:52.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:52.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:52 smithi150 ceph-mon[22676]: Deploying daemon osd.0 on smithi130 2024-04-22T21:06:53.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:53 smithi130 ceph-mon[21867]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:53.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:53 smithi130 ceph-mon[26394]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:53.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:53 smithi150 ceph-mon[22676]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:55.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:55 smithi130 ceph-mon[26394]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:55.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:55 smithi130 ceph-mon[21867]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:55.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:55 smithi150 ceph-mon[22676]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:57.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:57 smithi130 ceph-mon[21867]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:57.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:57.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:57.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:57.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:57.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:57 smithi130 ceph-mon[26394]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:57.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:57 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:57.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:57 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:57.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:57 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:57.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:57 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:57.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:57 smithi150 ceph-mon[22676]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:57.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:57 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:57.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:57 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:06:57.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:57 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:06:57.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:57 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:06:59.357 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 0 on host 'smithi130' 2024-04-22T21:06:59.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:59 smithi130 ceph-mon[21867]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:59.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:06:59 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:59.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:59 smithi130 ceph-mon[26394]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:59.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:06:59 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:06:59.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:59 smithi150 ceph-mon[22676]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:06:59.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:06:59 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:00.389 DEBUG:teuthology.orchestra.run.smithi130:osd.0> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.0.service 2024-04-22T21:07:00.392 INFO:tasks.cephadm:Deploying osd.1 on smithi130 with /dev/vg_nvme/lv_3... 2024-04-22T21:07:00.392 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-22T21:07:00.725 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:00 smithi130 ceph-mon[26394]: from='osd.0 [v2:172.21.15.130:6802/1555952224,v1:172.21.15.130:6803/1555952224]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T21:07:00.726 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:00 smithi130 ceph-mon[26394]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T21:07:00.726 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:00 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:00.726 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:00 smithi130 ceph-mon[21867]: from='osd.0 [v2:172.21.15.130:6802/1555952224,v1:172.21.15.130:6803/1555952224]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T21:07:00.726 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:00 smithi130 ceph-mon[21867]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T21:07:00.726 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:00 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:00.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:00 smithi150 ceph-mon[22676]: from='osd.0 [v2:172.21.15.130:6802/1555952224,v1:172.21.15.130:6803/1555952224]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T21:07:00.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:00 smithi150 ceph-mon[22676]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T21:07:00.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:00 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:01.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-mon[21867]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:07:01.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-mon[21867]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T21:07:01.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-mon[21867]: osdmap e6: 1 total, 0 up, 1 in 2024-04-22T21:07:01.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:01.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-mon[21867]: from='osd.0 [v2:172.21.15.130:6802/1555952224,v1:172.21.15.130:6803/1555952224]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:01.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-mon[21867]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:01.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-mon[26394]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:07:01.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-mon[26394]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T21:07:01.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-mon[26394]: osdmap e6: 1 total, 0 up, 1 in 2024-04-22T21:07:01.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:01.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-mon[26394]: from='osd.0 [v2:172.21.15.130:6802/1555952224,v1:172.21.15.130:6803/1555952224]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:01.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-mon[26394]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:01.810 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0[29273]: 2024-04-22T21:07:01.546+0000 7f2ed88ca700 -1 osd.0 0 waiting for initial osdmap 2024-04-22T21:07:01.810 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:07:01 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0[29273]: 2024-04-22T21:07:01.548+0000 7f2ed0a5c700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:07:01.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:01 smithi150 ceph-mon[22676]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:07:01.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:01 smithi150 ceph-mon[22676]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T21:07:01.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:01 smithi150 ceph-mon[22676]: osdmap e6: 1 total, 0 up, 1 in 2024-04-22T21:07:01.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:01 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:01.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:01 smithi150 ceph-mon[22676]: from='osd.0 [v2:172.21.15.130:6802/1555952224,v1:172.21.15.130:6803/1555952224]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:01.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:01 smithi150 ceph-mon[22676]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:02.477 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-22T21:07:02.478 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-22T21:07:02.478 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2024-04-22T21:07:02.478 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2024-04-22T21:07:02.478 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0121842 s, 861 MB/s 2024-04-22T21:07:02.478 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2024-04-22T21:07:02.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:02 smithi130 ceph-mon[21867]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-22T21:07:02.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:02 smithi130 ceph-mon[21867]: osdmap e7: 1 total, 0 up, 1 in 2024-04-22T21:07:02.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:02 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:02.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:02 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:02.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:02 smithi130 ceph-mon[26394]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-22T21:07:02.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:02 smithi130 ceph-mon[26394]: osdmap e7: 1 total, 0 up, 1 in 2024-04-22T21:07:02.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:02 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:02.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:02 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:02.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:02 smithi150 ceph-mon[22676]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-22T21:07:02.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:02 smithi150 ceph-mon[22676]: osdmap e7: 1 total, 0 up, 1 in 2024-04-22T21:07:02.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:02 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:02.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:02 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:03.627 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[21867]: purged_snaps scrub starts 2024-04-22T21:07:03.627 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[21867]: purged_snaps scrub ok 2024-04-22T21:07:03.627 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[21867]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:07:03.627 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:03.627 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[21867]: osd.0 [v2:172.21.15.130:6802/1555952224,v1:172.21.15.130:6803/1555952224] boot 2024-04-22T21:07:03.628 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[21867]: osdmap e8: 1 total, 1 up, 1 in 2024-04-22T21:07:03.628 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:03.628 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[26394]: purged_snaps scrub starts 2024-04-22T21:07:03.628 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[26394]: purged_snaps scrub ok 2024-04-22T21:07:03.628 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[26394]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:07:03.628 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:03.628 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[26394]: osd.0 [v2:172.21.15.130:6802/1555952224,v1:172.21.15.130:6803/1555952224] boot 2024-04-22T21:07:03.628 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[26394]: osdmap e8: 1 total, 1 up, 1 in 2024-04-22T21:07:03.628 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:03 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:03.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:03 smithi150 ceph-mon[22676]: purged_snaps scrub starts 2024-04-22T21:07:03.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:03 smithi150 ceph-mon[22676]: purged_snaps scrub ok 2024-04-22T21:07:03.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:03 smithi150 ceph-mon[22676]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T21:07:03.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:03 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:03.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:03 smithi150 ceph-mon[22676]: osd.0 [v2:172.21.15.130:6802/1555952224,v1:172.21.15.130:6803/1555952224] boot 2024-04-22T21:07:03.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:03 smithi150 ceph-mon[22676]: osdmap e8: 1 total, 1 up, 1 in 2024-04-22T21:07:03.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:03 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:07:04.233 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_3 2024-04-22T21:07:04.758 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:04 smithi130 ceph-mon[21867]: osdmap e9: 1 total, 1 up, 1 in 2024-04-22T21:07:04.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:04 smithi130 ceph-mon[26394]: osdmap e9: 1 total, 1 up, 1 in 2024-04-22T21:07:04.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:04 smithi150 ceph-mon[22676]: osdmap e9: 1 total, 1 up, 1 in 2024-04-22T21:07:05.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:05 smithi130 ceph-mon[26394]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:05.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:05 smithi130 ceph-mon[21867]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:05.829 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:05 smithi150 ceph-mon[22676]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:07.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[26394]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:07.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[26394]: Detected new or changed devices on smithi130 2024-04-22T21:07:07.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:07.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:07.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[26394]: Adjusting osd_memory_target on smithi130 to 16064M 2024-04-22T21:07:07.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:07.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:07.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:07.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:07.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:07.061 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:07.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[21867]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:07.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[21867]: Detected new or changed devices on smithi130 2024-04-22T21:07:07.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:07.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:07.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[21867]: Adjusting osd_memory_target on smithi130 to 16064M 2024-04-22T21:07:07.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:07.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:07.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:07.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:07.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:07.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:06 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:07.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:06 smithi150 ceph-mon[22676]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:07.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:06 smithi150 ceph-mon[22676]: Detected new or changed devices on smithi130 2024-04-22T21:07:07.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:06 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:07.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:06 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:07.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:06 smithi150 ceph-mon[22676]: Adjusting osd_memory_target on smithi130 to 16064M 2024-04-22T21:07:07.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:06 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:07.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:06 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:07.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:06 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:07.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:06 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:07.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:06 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:07.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:06 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:07.930 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:07 smithi130 ceph-mon[26394]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:07:07.931 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:07 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:07:07.931 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:07 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:07:07.931 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:07 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:07.931 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:07 smithi130 ceph-mon[21867]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:07:07.931 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:07 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:07:07.931 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:07 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:07:07.931 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:07 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:07 smithi150 ceph-mon[22676]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:07:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:07 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:07:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:07 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:07:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:07 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:08.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:08 smithi130 ceph-mon[21867]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:09.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:08 smithi130 ceph-mon[26394]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:09.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:08 smithi150 ceph-mon[22676]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:10.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:09 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3064376721' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "189889ca-c6b5-4c2c-a3f8-082e23fe4669"}]: dispatch 2024-04-22T21:07:10.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:09 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3064376721' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "189889ca-c6b5-4c2c-a3f8-082e23fe4669"}]': finished 2024-04-22T21:07:10.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:09 smithi130 ceph-mon[21867]: osdmap e10: 2 total, 1 up, 2 in 2024-04-22T21:07:10.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:09 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:10.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:09 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2059281124' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:07:10.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:09 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3064376721' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "189889ca-c6b5-4c2c-a3f8-082e23fe4669"}]: dispatch 2024-04-22T21:07:10.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:09 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3064376721' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "189889ca-c6b5-4c2c-a3f8-082e23fe4669"}]': finished 2024-04-22T21:07:10.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:09 smithi130 ceph-mon[26394]: osdmap e10: 2 total, 1 up, 2 in 2024-04-22T21:07:10.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:09 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:10.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:09 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2059281124' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:07:10.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:09 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3064376721' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "189889ca-c6b5-4c2c-a3f8-082e23fe4669"}]: dispatch 2024-04-22T21:07:10.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:09 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3064376721' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "189889ca-c6b5-4c2c-a3f8-082e23fe4669"}]': finished 2024-04-22T21:07:10.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:09 smithi150 ceph-mon[22676]: osdmap e10: 2 total, 1 up, 2 in 2024-04-22T21:07:10.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:09 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:10.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:09 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2059281124' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:07:11.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:11 smithi130 ceph-mon[21867]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:11.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:11 smithi130 ceph-mon[26394]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:11.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:11 smithi150 ceph-mon[22676]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:13.284 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:13 smithi130 ceph-mon[21867]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:13.284 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:13 smithi130 ceph-mon[26394]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:13.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:13 smithi150 ceph-mon[22676]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:15.410 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:15 smithi130 ceph-mon[26394]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:15.410 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:15 smithi130 ceph-mon[21867]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:15.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:15 smithi150 ceph-mon[22676]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:16.514 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:16 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T21:07:16.514 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:16 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:16.514 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:16 smithi150 ceph-mon[22676]: Deploying daemon osd.1 on smithi130 2024-04-22T21:07:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:16 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T21:07:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:16 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:16 smithi130 ceph-mon[21867]: Deploying daemon osd.1 on smithi130 2024-04-22T21:07:16.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:16 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T21:07:16.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:16 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:16.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:16 smithi130 ceph-mon[26394]: Deploying daemon osd.1 on smithi130 2024-04-22T21:07:17.449 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:17 smithi130 ceph-mon[21867]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:17.449 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:17 smithi130 ceph-mon[26394]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:17.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:17 smithi150 ceph-mon[22676]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:19.481 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:19 smithi130 ceph-mon[26394]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:19.481 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:19 smithi130 ceph-mon[21867]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:19.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:19 smithi150 ceph-mon[22676]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:21.347 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:21 smithi130 ceph-mon[21867]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:21.347 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:21 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:21.347 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:21 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:21.347 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:21 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:21.347 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:21 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:21.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:21 smithi130 ceph-mon[26394]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:21.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:21 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:21.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:21 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:21.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:21 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:21.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:21 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:21.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:21 smithi150 ceph-mon[22676]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:21.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:21 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:21.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:21 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:21.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:21 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:21.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:21 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:23.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:23 smithi130 ceph-mon[21867]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:23.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:23 smithi130 ceph-mon[26394]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:23.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:23 smithi150 ceph-mon[22676]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:24.098 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 1 on host 'smithi130' 2024-04-22T21:07:24.462 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:24.462 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:24.462 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[26394]: from='osd.1 [v2:172.21.15.130:6810/456112775,v1:172.21.15.130:6811/456112775]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T21:07:24.462 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[26394]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T21:07:24.462 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:24.463 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:24.463 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:24.463 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:24.463 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:24.463 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:24.463 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[21867]: from='osd.1 [v2:172.21.15.130:6810/456112775,v1:172.21.15.130:6811/456112775]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T21:07:24.463 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[21867]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T21:07:24.463 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:24.464 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:24.464 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:24.464 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:24 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:24.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:24 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:24.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:24 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:24.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:24 smithi150 ceph-mon[22676]: from='osd.1 [v2:172.21.15.130:6810/456112775,v1:172.21.15.130:6811/456112775]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T21:07:24.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:24 smithi150 ceph-mon[22676]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T21:07:24.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:24 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:24.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:24 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:24.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:24 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:24.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:24 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:24.811 DEBUG:teuthology.orchestra.run.smithi130:osd.1> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.1.service 2024-04-22T21:07:24.813 INFO:tasks.cephadm:Deploying osd.2 on smithi130 with /dev/vg_nvme/lv_2... 2024-04-22T21:07:24.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-22T21:07:25.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-mon[21867]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:25.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-mon[21867]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T21:07:25.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-mon[21867]: osdmap e11: 2 total, 1 up, 2 in 2024-04-22T21:07:25.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:25.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-mon[21867]: from='osd.1 [v2:172.21.15.130:6810/456112775,v1:172.21.15.130:6811/456112775]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:25.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-mon[21867]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:25.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-mon[26394]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:25.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-mon[26394]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T21:07:25.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-mon[26394]: osdmap e11: 2 total, 1 up, 2 in 2024-04-22T21:07:25.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:25.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-mon[26394]: from='osd.1 [v2:172.21.15.130:6810/456112775,v1:172.21.15.130:6811/456112775]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:25.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-mon[26394]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:25.560 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1[32250]: 2024-04-22T21:07:25.282+0000 7f0739853700 -1 osd.1 0 waiting for initial osdmap 2024-04-22T21:07:25.560 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:07:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1[32250]: 2024-04-22T21:07:25.284+0000 7f07341ea700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:07:25.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:25 smithi150 ceph-mon[22676]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:25.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:25 smithi150 ceph-mon[22676]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T21:07:25.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:25 smithi150 ceph-mon[22676]: osdmap e11: 2 total, 1 up, 2 in 2024-04-22T21:07:25.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:25 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:25.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:25 smithi150 ceph-mon[22676]: from='osd.1 [v2:172.21.15.130:6810/456112775,v1:172.21.15.130:6811/456112775]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:25.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:25 smithi150 ceph-mon[22676]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:26.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:26 smithi130 ceph-mon[21867]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-22T21:07:26.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:26 smithi130 ceph-mon[21867]: osdmap e12: 2 total, 1 up, 2 in 2024-04-22T21:07:26.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:26 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:26.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:26 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:26.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:26 smithi130 ceph-mon[26394]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-22T21:07:26.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:26 smithi130 ceph-mon[26394]: osdmap e12: 2 total, 1 up, 2 in 2024-04-22T21:07:26.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:26 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:26.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:26 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:26.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:26 smithi150 ceph-mon[22676]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-22T21:07:26.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:26 smithi150 ceph-mon[22676]: osdmap e12: 2 total, 1 up, 2 in 2024-04-22T21:07:26.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:26 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:26.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:26 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:27.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[26394]: purged_snaps scrub starts 2024-04-22T21:07:27.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[26394]: purged_snaps scrub ok 2024-04-22T21:07:27.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[26394]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:27.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:27.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[26394]: osd.1 [v2:172.21.15.130:6810/456112775,v1:172.21.15.130:6811/456112775] boot 2024-04-22T21:07:27.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[26394]: osdmap e13: 2 total, 2 up, 2 in 2024-04-22T21:07:27.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:27.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[21867]: purged_snaps scrub starts 2024-04-22T21:07:27.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[21867]: purged_snaps scrub ok 2024-04-22T21:07:27.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[21867]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:27.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:27.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[21867]: osd.1 [v2:172.21.15.130:6810/456112775,v1:172.21.15.130:6811/456112775] boot 2024-04-22T21:07:27.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[21867]: osdmap e13: 2 total, 2 up, 2 in 2024-04-22T21:07:27.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:27.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:27 smithi150 ceph-mon[22676]: purged_snaps scrub starts 2024-04-22T21:07:27.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:27 smithi150 ceph-mon[22676]: purged_snaps scrub ok 2024-04-22T21:07:27.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:27 smithi150 ceph-mon[22676]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T21:07:27.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:27 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:27.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:27 smithi150 ceph-mon[22676]: osd.1 [v2:172.21.15.130:6810/456112775,v1:172.21.15.130:6811/456112775] boot 2024-04-22T21:07:27.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:27 smithi150 ceph-mon[22676]: osdmap e13: 2 total, 2 up, 2 in 2024-04-22T21:07:27.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:27 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:07:28.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:28 smithi130 ceph-mon[26394]: osdmap e14: 2 total, 2 up, 2 in 2024-04-22T21:07:28.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:28 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:28.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:28 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:28.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:28 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:28.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:28 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:28.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:28 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:28.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:28 smithi130 ceph-mon[21867]: osdmap e14: 2 total, 2 up, 2 in 2024-04-22T21:07:28.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:28 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:28.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:28 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:28.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:28 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:28.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:28 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:28.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:28 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:28.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:28 smithi150 ceph-mon[22676]: osdmap e14: 2 total, 2 up, 2 in 2024-04-22T21:07:28.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:28 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:28.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:28 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:28.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:28 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:28.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:28 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:28.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:28 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:28.749 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-22T21:07:28.749 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-22T21:07:28.749 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2024-04-22T21:07:28.749 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2024-04-22T21:07:28.749 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0127926 s, 820 MB/s 2024-04-22T21:07:28.750 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2024-04-22T21:07:29.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:29 smithi130 ceph-mon[21867]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:29.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:29 smithi130 ceph-mon[21867]: Detected new or changed devices on smithi130 2024-04-22T21:07:29.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:29 smithi130 ceph-mon[21867]: Adjusting osd_memory_target on smithi130 to 8032M 2024-04-22T21:07:29.389 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_2 2024-04-22T21:07:29.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:29 smithi130 ceph-mon[26394]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:29.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:29 smithi130 ceph-mon[26394]: Detected new or changed devices on smithi130 2024-04-22T21:07:29.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:29 smithi130 ceph-mon[26394]: Adjusting osd_memory_target on smithi130 to 8032M 2024-04-22T21:07:29.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:29 smithi150 ceph-mon[22676]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:29.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:29 smithi150 ceph-mon[22676]: Detected new or changed devices on smithi130 2024-04-22T21:07:29.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:29 smithi150 ceph-mon[22676]: Adjusting osd_memory_target on smithi130 to 8032M 2024-04-22T21:07:31.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:31 smithi130 ceph-mon[21867]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:31.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:31 smithi130 ceph-mon[26394]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:31.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:31 smithi150 ceph-mon[22676]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:32.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:32 smithi130 ceph-mon[21867]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:07:32.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:32 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:07:32.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:32 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:07:32.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:32 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:32.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:32 smithi130 ceph-mon[26394]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:07:32.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:32 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:07:32.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:32 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:07:32.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:32 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:32.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:32 smithi150 ceph-mon[22676]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:07:32.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:32 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:07:32.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:32 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:07:32.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:32 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:33.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:33 smithi130 ceph-mon[21867]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:33.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:33 smithi130 ceph-mon[26394]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:33.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:33 smithi150 ceph-mon[22676]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:34.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:34 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3120778064' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1993302f-f436-47e7-8cf5-7e00c0bdd1db"}]: dispatch 2024-04-22T21:07:34.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:34 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3120778064' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1993302f-f436-47e7-8cf5-7e00c0bdd1db"}]': finished 2024-04-22T21:07:34.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:34 smithi130 ceph-mon[26394]: osdmap e15: 3 total, 2 up, 3 in 2024-04-22T21:07:34.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:34 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:34.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:34 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1402092440' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:07:34.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:34 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3120778064' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1993302f-f436-47e7-8cf5-7e00c0bdd1db"}]: dispatch 2024-04-22T21:07:34.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:34 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3120778064' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1993302f-f436-47e7-8cf5-7e00c0bdd1db"}]': finished 2024-04-22T21:07:34.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:34 smithi130 ceph-mon[21867]: osdmap e15: 3 total, 2 up, 3 in 2024-04-22T21:07:34.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:34 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:34.560 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:34 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1402092440' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:07:34.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:34 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3120778064' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1993302f-f436-47e7-8cf5-7e00c0bdd1db"}]: dispatch 2024-04-22T21:07:34.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:34 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3120778064' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1993302f-f436-47e7-8cf5-7e00c0bdd1db"}]': finished 2024-04-22T21:07:34.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:34 smithi150 ceph-mon[22676]: osdmap e15: 3 total, 2 up, 3 in 2024-04-22T21:07:34.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:34 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:34.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:34 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1402092440' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:07:36.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:35 smithi130 ceph-mon[26394]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:36.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:35 smithi130 ceph-mon[21867]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:36.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:35 smithi150 ceph-mon[22676]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:37.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:37 smithi130 ceph-mon[21867]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:37.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:37 smithi130 ceph-mon[26394]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:38.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:37 smithi150 ceph-mon[22676]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:40.011 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:39 smithi130 ceph-mon[26394]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:40.012 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:39 smithi130 ceph-mon[21867]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:40.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:39 smithi150 ceph-mon[22676]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:40.764 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:40 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T21:07:40.764 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:40 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:40.765 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:40 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T21:07:40.765 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:40 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:41.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:40 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T21:07:41.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:40 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:41.692 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:41 smithi130 ceph-mon[21867]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:41.692 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:41 smithi130 ceph-mon[21867]: Deploying daemon osd.2 on smithi130 2024-04-22T21:07:41.693 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:41 smithi130 ceph-mon[26394]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:41.693 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:41 smithi130 ceph-mon[26394]: Deploying daemon osd.2 on smithi130 2024-04-22T21:07:42.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:41 smithi150 ceph-mon[22676]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:42.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:41 smithi150 ceph-mon[22676]: Deploying daemon osd.2 on smithi130 2024-04-22T21:07:42.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:07:42.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:07:42.683 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:42 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:07:42.683 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:42 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:07:43.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:42 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:07:43.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:42 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:07:43.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:43 smithi130 ceph-mon[21867]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:43.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:43 smithi130 ceph-mon[26394]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:44.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:43 smithi150 ceph-mon[22676]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:45.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:45 smithi130 ceph-mon[21867]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:45.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:45 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:45.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:45 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:45.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:45 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:45.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:45 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:45.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:45 smithi130 ceph-mon[26394]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:45.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:45 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:45.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:45 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:45.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:45 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:45.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:45 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:46.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:45 smithi150 ceph-mon[22676]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:46.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:45 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:46.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:45 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:46.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:45 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:46.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:45 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:47.790 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:47 smithi130 ceph-mon[21867]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:47.791 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:47 smithi130 ceph-mon[26394]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:47.915 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:47 smithi150 ceph-mon[22676]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:48.633 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 2 on host 'smithi130' 2024-04-22T21:07:48.997 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[21867]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:48.998 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:48.998 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:48.998 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[21867]: from='osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T21:07:48.998 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:48.998 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:48.998 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:48.998 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:48.999 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[26394]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:48.999 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:48.999 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:48.999 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[26394]: from='osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T21:07:48.999 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:48.999 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:48.999 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:48.999 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:48 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:49.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:48 smithi150 ceph-mon[22676]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:49.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:48 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:49.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:48 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:49.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:48 smithi150 ceph-mon[22676]: from='osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T21:07:49.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:48 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:49.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:48 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:07:49.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:48 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:49.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:48 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:07:49.331 DEBUG:teuthology.orchestra.run.smithi130:osd.2> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.2.service 2024-04-22T21:07:49.334 INFO:tasks.cephadm:Deploying osd.3 on smithi130 with /dev/vg_nvme/lv_1... 2024-04-22T21:07:49.334 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-22T21:07:50.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:49 smithi130 ceph-mon[21867]: from='osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T21:07:50.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:49 smithi130 ceph-mon[21867]: osdmap e16: 3 total, 2 up, 3 in 2024-04-22T21:07:50.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:49 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:50.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:49 smithi130 ceph-mon[21867]: from='osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:50.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:49 smithi130 ceph-mon[26394]: from='osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T21:07:50.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:49 smithi130 ceph-mon[26394]: osdmap e16: 3 total, 2 up, 3 in 2024-04-22T21:07:50.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:49 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:50.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:49 smithi130 ceph-mon[26394]: from='osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:50.060 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:07:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2[35178]: 2024-04-22T21:07:49.818+0000 7f21f810e700 -1 osd.2 0 waiting for initial osdmap 2024-04-22T21:07:50.060 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:07:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2[35178]: 2024-04-22T21:07:49.822+0000 7f21f2aa5700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:07:50.074 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:49 smithi150 ceph-mon[22676]: from='osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T21:07:50.074 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:49 smithi150 ceph-mon[22676]: osdmap e16: 3 total, 2 up, 3 in 2024-04-22T21:07:50.074 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:49 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:50.074 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:49 smithi150 ceph-mon[22676]: from='osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:07:51.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:50 smithi130 ceph-mon[26394]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:51.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:50 smithi130 ceph-mon[26394]: from='osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-22T21:07:51.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:50 smithi130 ceph-mon[26394]: osdmap e17: 3 total, 2 up, 3 in 2024-04-22T21:07:51.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:50 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:51.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:50 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:51.035 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:50 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:51.035 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:50 smithi130 ceph-mon[21867]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:51.035 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:50 smithi130 ceph-mon[21867]: from='osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-22T21:07:51.035 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:50 smithi130 ceph-mon[21867]: osdmap e17: 3 total, 2 up, 3 in 2024-04-22T21:07:51.036 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:50 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:51.036 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:50 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:51.036 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:50 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:51.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:50 smithi150 ceph-mon[22676]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:51.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:50 smithi150 ceph-mon[22676]: from='osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-22T21:07:51.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:50 smithi150 ceph-mon[22676]: osdmap e17: 3 total, 2 up, 3 in 2024-04-22T21:07:51.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:50 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:51.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:50 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:51.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:50 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:52.243 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:52 smithi130 ceph-mon[26394]: purged_snaps scrub starts 2024-04-22T21:07:52.243 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:52 smithi130 ceph-mon[26394]: purged_snaps scrub ok 2024-04-22T21:07:52.243 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:52 smithi130 ceph-mon[26394]: osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272] boot 2024-04-22T21:07:52.243 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:52 smithi130 ceph-mon[26394]: osdmap e18: 3 total, 3 up, 3 in 2024-04-22T21:07:52.243 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:52.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:52 smithi130 ceph-mon[21867]: purged_snaps scrub starts 2024-04-22T21:07:52.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:52 smithi130 ceph-mon[21867]: purged_snaps scrub ok 2024-04-22T21:07:52.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:52 smithi130 ceph-mon[21867]: osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272] boot 2024-04-22T21:07:52.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:52 smithi130 ceph-mon[21867]: osdmap e18: 3 total, 3 up, 3 in 2024-04-22T21:07:52.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:52.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:52 smithi150 ceph-mon[22676]: purged_snaps scrub starts 2024-04-22T21:07:52.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:52 smithi150 ceph-mon[22676]: purged_snaps scrub ok 2024-04-22T21:07:52.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:52 smithi150 ceph-mon[22676]: osd.2 [v2:172.21.15.130:6818/802772272,v1:172.21.15.130:6819/802772272] boot 2024-04-22T21:07:52.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:52 smithi150 ceph-mon[22676]: osdmap e18: 3 total, 3 up, 3 in 2024-04-22T21:07:52.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:07:53.354 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-22T21:07:53.354 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-22T21:07:53.354 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2024-04-22T21:07:53.355 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2024-04-22T21:07:53.355 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112566 s, 932 MB/s 2024-04-22T21:07:53.355 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2024-04-22T21:07:53.355 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[21867]: pgmap v73: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:53.355 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-22T21:07:53.355 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-22T21:07:53.355 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[21867]: osdmap e19: 3 total, 3 up, 3 in 2024-04-22T21:07:53.355 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-22T21:07:53.355 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[21867]: Detected new or changed devices on smithi130 2024-04-22T21:07:53.355 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:53.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:53.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:53.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:53.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[21867]: Adjusting osd_memory_target on smithi130 to 5354M 2024-04-22T21:07:53.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:53.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:53.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[26394]: pgmap v73: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:53.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-22T21:07:53.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-22T21:07:53.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[26394]: osdmap e19: 3 total, 3 up, 3 in 2024-04-22T21:07:53.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-22T21:07:53.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[26394]: Detected new or changed devices on smithi130 2024-04-22T21:07:53.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:53.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:53.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:53.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:53.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[26394]: Adjusting osd_memory_target on smithi130 to 5354M 2024-04-22T21:07:53.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:53.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:53 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:53.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:53 smithi150 ceph-mon[22676]: pgmap v73: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T21:07:53.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:53 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-22T21:07:53.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:53 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-22T21:07:53.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:53 smithi150 ceph-mon[22676]: osdmap e19: 3 total, 3 up, 3 in 2024-04-22T21:07:53.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:53 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-22T21:07:53.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:53 smithi150 ceph-mon[22676]: Detected new or changed devices on smithi130 2024-04-22T21:07:53.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:53 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:53.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:53 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:53.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:53 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:53.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:53 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:07:53.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:53 smithi150 ceph-mon[22676]: Adjusting osd_memory_target on smithi130 to 5354M 2024-04-22T21:07:53.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:53 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:53.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:53 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:54.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_1 2024-04-22T21:07:54.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:54 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-22T21:07:54.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:54 smithi130 ceph-mon[21867]: osdmap e20: 3 total, 3 up, 3 in 2024-04-22T21:07:54.306 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:07:54 smithi130 sudo[36411]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T21:07:54.307 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:07:54 smithi130 sudo[36411]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T21:07:54.307 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:07:54 smithi130 sudo[36411]: pam_unix(sudo:session): session closed for user root 2024-04-22T21:07:54.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:54 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-22T21:07:54.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:54 smithi130 ceph-mon[26394]: osdmap e20: 3 total, 3 up, 3 in 2024-04-22T21:07:54.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:54 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-22T21:07:54.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:54 smithi150 ceph-mon[22676]: osdmap e20: 3 total, 3 up, 3 in 2024-04-22T21:07:54.558 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:07:54 smithi130 sudo[36432]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T21:07:54.559 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:07:54 smithi130 sudo[36432]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T21:07:54.559 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:07:54 smithi130 sudo[36432]: pam_unix(sudo:session): session closed for user root 2024-04-22T21:07:54.850 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:07:54 smithi130 sudo[36461]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T21:07:54.851 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:07:54 smithi130 sudo[36461]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T21:07:54.851 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:07:54 smithi130 sudo[36461]: pam_unix(sudo:session): session closed for user root 2024-04-22T21:07:55.102 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:07:54 smithi130 sudo[36481]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T21:07:55.102 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:07:54 smithi130 sudo[36481]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T21:07:55.102 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:07:54 smithi130 sudo[36481]: pam_unix(sudo:session): session closed for user root 2024-04-22T21:07:55.103 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:55 smithi130 ceph-mon[21867]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:07:55.103 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:55 smithi130 ceph-mon[21867]: osdmap e21: 3 total, 3 up, 3 in 2024-04-22T21:07:55.103 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:55 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:55.103 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:55 smithi130 ceph-mon[26394]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:07:55.103 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:55 smithi130 ceph-mon[26394]: osdmap e21: 3 total, 3 up, 3 in 2024-04-22T21:07:55.103 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:55 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:55.359 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:07:55 smithi130 sudo[36484]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T21:07:55.360 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:07:55 smithi130 sudo[36484]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T21:07:55.360 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:07:55 smithi130 sudo[36484]: pam_unix(sudo:session): session closed for user root 2024-04-22T21:07:55.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:55 smithi150 ceph-mon[22676]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:07:55.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:55 smithi150 ceph-mon[22676]: osdmap e21: 3 total, 3 up, 3 in 2024-04-22T21:07:55.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:55 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:55.611 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:07:55 smithi130 sudo[36490]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T21:07:55.611 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:07:55 smithi130 sudo[36490]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T21:07:55.611 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:07:55 smithi130 sudo[36490]: pam_unix(sudo:session): session closed for user root 2024-04-22T21:07:55.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:55 smithi130 sudo[36511]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T21:07:55.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:55 smithi130 sudo[36511]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T21:07:55.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:55 smithi130 sudo[36511]: pam_unix(sudo:session): session closed for user root 2024-04-22T21:07:56.123 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:55 smithi130 sudo[36519]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T21:07:56.123 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:55 smithi130 sudo[36519]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T21:07:56.123 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:55 smithi130 sudo[36519]: pam_unix(sudo:session): session closed for user root 2024-04-22T21:07:56.385 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 sudo[36544]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T21:07:56.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 sudo[36544]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T21:07:56.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 sudo[36544]: pam_unix(sudo:session): session closed for user root 2024-04-22T21:07:56.679 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 sudo[36586]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T21:07:56.679 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 sudo[36586]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T21:07:56.679 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 sudo[36586]: pam_unix(sudo:session): session closed for user root 2024-04-22T21:07:56.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 sudo[24192]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T21:07:56.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 sudo[24192]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T21:07:56.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 sudo[24192]: pam_unix(sudo:session): session closed for user root 2024-04-22T21:07:56.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 sudo[24195]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T21:07:56.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 sudo[24195]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T21:07:56.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 sudo[24195]: pam_unix(sudo:session): session closed for user root 2024-04-22T21:07:57.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[21867]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T21:07:57.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[21867]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T21:07:57.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:07:57.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:07:57.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:07:57.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:07:57.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:07:57.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:07:57.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[26394]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T21:07:57.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[26394]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T21:07:57.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:07:57.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:07:57.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:07:57.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:07:57.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:07:57.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:56 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:07:57.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 ceph-mon[22676]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T21:07:57.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 ceph-mon[22676]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T21:07:57.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:07:57.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:07:57.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:07:57.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:07:57.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:07:57.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:56 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:07:58.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:07:58.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T21:07:58.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: from='client.14313 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:07:58.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:07:58.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:07:58.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:58.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T21:07:58.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T21:07:58.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:07:58.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:07:58.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:07:58.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T21:07:58.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-22T21:07:58.310 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:58.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:07:58.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T21:07:58.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: from='client.14313 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:07:58.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:07:58.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:07:58.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:58.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T21:07:58.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T21:07:58.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:07:58.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:07:58.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:07:58.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T21:07:58.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-22T21:07:58.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:57 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:58.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:07:58.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T21:07:58.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: from='client.14313 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:07:58.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:07:58.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:07:58.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:07:58.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T21:07:58.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T21:07:58.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:07:58.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:07:58.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:07:58.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T21:07:58.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-22T21:07:58.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:57 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:07:59.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:58 smithi130 ceph-mon[26394]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:07:59.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:58 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/374502700' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9093e7a3-e43a-4387-bb67-c5f6eff53bae"}]: dispatch 2024-04-22T21:07:59.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:58 smithi130 ceph-mon[26394]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9093e7a3-e43a-4387-bb67-c5f6eff53bae"}]: dispatch 2024-04-22T21:07:59.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:58 smithi130 ceph-mon[26394]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9093e7a3-e43a-4387-bb67-c5f6eff53bae"}]': finished 2024-04-22T21:07:59.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:58 smithi130 ceph-mon[26394]: osdmap e22: 4 total, 3 up, 4 in 2024-04-22T21:07:59.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:58 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:07:59.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:58 smithi130 ceph-mon[21867]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:07:59.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:58 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/374502700' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9093e7a3-e43a-4387-bb67-c5f6eff53bae"}]: dispatch 2024-04-22T21:07:59.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:58 smithi130 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9093e7a3-e43a-4387-bb67-c5f6eff53bae"}]: dispatch 2024-04-22T21:07:59.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:58 smithi130 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9093e7a3-e43a-4387-bb67-c5f6eff53bae"}]': finished 2024-04-22T21:07:59.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:58 smithi130 ceph-mon[21867]: osdmap e22: 4 total, 3 up, 4 in 2024-04-22T21:07:59.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:58 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:07:59.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:58 smithi150 ceph-mon[22676]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:07:59.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:58 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/374502700' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9093e7a3-e43a-4387-bb67-c5f6eff53bae"}]: dispatch 2024-04-22T21:07:59.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:58 smithi150 ceph-mon[22676]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9093e7a3-e43a-4387-bb67-c5f6eff53bae"}]: dispatch 2024-04-22T21:07:59.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:58 smithi150 ceph-mon[22676]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9093e7a3-e43a-4387-bb67-c5f6eff53bae"}]': finished 2024-04-22T21:07:59.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:58 smithi150 ceph-mon[22676]: osdmap e22: 4 total, 3 up, 4 in 2024-04-22T21:07:59.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:58 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:08:00.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:07:59 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1069686923' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:08:00.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:07:59 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1069686923' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:08:00.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:07:59 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1069686923' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:08:01.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:00 smithi130 ceph-mon[21867]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:01.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:00 smithi130 ceph-mon[26394]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:01.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:00 smithi150 ceph-mon[22676]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:03.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:02 smithi130 ceph-mon[21867]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:03.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:02 smithi130 ceph-mon[26394]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:03.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:02 smithi150 ceph-mon[22676]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:05.225 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:04 smithi130 ceph-mon[26394]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:05.225 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:04 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T21:08:05.225 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:04 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:05.225 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:04 smithi130 ceph-mon[26394]: Deploying daemon osd.3 on smithi130 2024-04-22T21:08:05.226 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:04 smithi130 ceph-mon[21867]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:05.226 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:04 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T21:08:05.226 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:04 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:05.226 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:04 smithi130 ceph-mon[21867]: Deploying daemon osd.3 on smithi130 2024-04-22T21:08:05.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:04 smithi150 ceph-mon[22676]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:05.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:04 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T21:08:05.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:04 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:05.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:04 smithi150 ceph-mon[22676]: Deploying daemon osd.3 on smithi130 2024-04-22T21:08:07.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:06 smithi130 ceph-mon[21867]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:07.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:06 smithi130 ceph-mon[26394]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:07.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:06 smithi150 ceph-mon[22676]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:09.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:08 smithi130 ceph-mon[21867]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:09.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:08 smithi130 ceph-mon[26394]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:09.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:08 smithi150 ceph-mon[22676]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:10 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:10 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:10 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:10 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:10 smithi130 ceph-mon[21867]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:11.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:10 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:11.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:10 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:11.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:10 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:11.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:10 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:11.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:10 smithi130 ceph-mon[26394]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:11.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:10 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:11.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:10 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:11.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:10 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:11.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:10 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:11.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:10 smithi150 ceph-mon[22676]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:13.197 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 3 on host 'smithi130' 2024-04-22T21:08:13.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[21867]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:13.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:13.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[21867]: from='osd.3 [v2:172.21.15.130:6826/929040654,v1:172.21.15.130:6827/929040654]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T21:08:13.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[21867]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T21:08:13.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:13.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:13.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:13.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:13.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:13.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[26394]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:13.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:13.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[26394]: from='osd.3 [v2:172.21.15.130:6826/929040654,v1:172.21.15.130:6827/929040654]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T21:08:13.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[26394]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T21:08:13.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:13.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:13.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:13.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:13.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:13 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:13.817 DEBUG:teuthology.orchestra.run.smithi130:osd.3> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.3.service 2024-04-22T21:08:13.819 INFO:tasks.cephadm:Deploying osd.4 on smithi150 with /dev/vg_nvme/lv_4... 2024-04-22T21:08:13.820 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-22T21:08:13.845 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:13 smithi150 ceph-mon[22676]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:13.845 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:13 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:13.845 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:13 smithi150 ceph-mon[22676]: from='osd.3 [v2:172.21.15.130:6826/929040654,v1:172.21.15.130:6827/929040654]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T21:08:13.846 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:13 smithi150 ceph-mon[22676]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T21:08:13.846 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:13 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:13.846 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:13 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:13.846 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:13 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:13.846 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:13 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:13.846 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:13 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:14.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:14 smithi130 ceph-mon[21867]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T21:08:14.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:14 smithi130 ceph-mon[21867]: osdmap e23: 4 total, 3 up, 4 in 2024-04-22T21:08:14.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:14 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:08:14.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:14 smithi130 ceph-mon[21867]: from='osd.3 [v2:172.21.15.130:6826/929040654,v1:172.21.15.130:6827/929040654]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:08:14.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:14 smithi130 ceph-mon[21867]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:08:14.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:14 smithi130 ceph-mon[26394]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T21:08:14.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:14 smithi130 ceph-mon[26394]: osdmap e23: 4 total, 3 up, 4 in 2024-04-22T21:08:14.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:14 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:08:14.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:14 smithi130 ceph-mon[26394]: from='osd.3 [v2:172.21.15.130:6826/929040654,v1:172.21.15.130:6827/929040654]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:08:14.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:14 smithi130 ceph-mon[26394]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:08:14.810 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:08:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3[38170]: 2024-04-22T21:08:14.473+0000 7fd733988700 -1 osd.3 0 waiting for initial osdmap 2024-04-22T21:08:14.811 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:08:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3[38170]: 2024-04-22T21:08:14.477+0000 7fd72db1e700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:08:14.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:14 smithi150 ceph-mon[22676]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T21:08:14.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:14 smithi150 ceph-mon[22676]: osdmap e23: 4 total, 3 up, 4 in 2024-04-22T21:08:14.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:14 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:08:14.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:14 smithi150 ceph-mon[22676]: from='osd.3 [v2:172.21.15.130:6826/929040654,v1:172.21.15.130:6827/929040654]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:08:14.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:14 smithi150 ceph-mon[22676]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:08:15.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:15 smithi150 ceph-mon[22676]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:15.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:15 smithi150 ceph-mon[22676]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-22T21:08:15.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:15 smithi150 ceph-mon[22676]: osdmap e24: 4 total, 3 up, 4 in 2024-04-22T21:08:15.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:15 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:08:15.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:15 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:08:15.751 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-22T21:08:15.751 INFO:teuthology.orchestra.run.smithi150.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-22T21:08:15.751 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10+0 records in 2024-04-22T21:08:15.752 INFO:teuthology.orchestra.run.smithi150.stderr:10+0 records out 2024-04-22T21:08:15.752 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112154 s, 935 MB/s 2024-04-22T21:08:15.752 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping successful for: 2024-04-22T21:08:15.781 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:15 smithi130 ceph-mon[21867]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:15.781 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:15 smithi130 ceph-mon[21867]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-22T21:08:15.781 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:15 smithi130 ceph-mon[21867]: osdmap e24: 4 total, 3 up, 4 in 2024-04-22T21:08:15.781 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:15 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:08:15.781 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:15 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:08:15.781 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:15 smithi130 ceph-mon[26394]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:15.781 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:15 smithi130 ceph-mon[26394]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-04-22T21:08:15.782 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:15 smithi130 ceph-mon[26394]: osdmap e24: 4 total, 3 up, 4 in 2024-04-22T21:08:15.782 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:15 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:08:15.782 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:15 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:08:16.325 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi150:vg_nvme/lv_4 2024-04-22T21:08:16.662 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:16 smithi150 ceph-mon[22676]: purged_snaps scrub starts 2024-04-22T21:08:16.662 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:16 smithi150 ceph-mon[22676]: purged_snaps scrub ok 2024-04-22T21:08:16.662 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:16 smithi150 ceph-mon[22676]: osd.3 [v2:172.21.15.130:6826/929040654,v1:172.21.15.130:6827/929040654] boot 2024-04-22T21:08:16.662 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:16 smithi150 ceph-mon[22676]: osdmap e25: 4 total, 4 up, 4 in 2024-04-22T21:08:16.662 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:16 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:08:16.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:16 smithi130 ceph-mon[21867]: purged_snaps scrub starts 2024-04-22T21:08:16.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:16 smithi130 ceph-mon[21867]: purged_snaps scrub ok 2024-04-22T21:08:16.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:16 smithi130 ceph-mon[21867]: osd.3 [v2:172.21.15.130:6826/929040654,v1:172.21.15.130:6827/929040654] boot 2024-04-22T21:08:16.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:16 smithi130 ceph-mon[21867]: osdmap e25: 4 total, 4 up, 4 in 2024-04-22T21:08:16.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:16 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:08:16.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:16 smithi130 ceph-mon[26394]: purged_snaps scrub starts 2024-04-22T21:08:16.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:16 smithi130 ceph-mon[26394]: purged_snaps scrub ok 2024-04-22T21:08:16.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:16 smithi130 ceph-mon[26394]: osd.3 [v2:172.21.15.130:6826/929040654,v1:172.21.15.130:6827/929040654] boot 2024-04-22T21:08:16.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:16 smithi130 ceph-mon[26394]: osdmap e25: 4 total, 4 up, 4 in 2024-04-22T21:08:16.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:16 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:08:17.592 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:17 smithi150 ceph-mon[22676]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:17.592 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:17 smithi150 ceph-mon[22676]: osdmap e26: 4 total, 4 up, 4 in 2024-04-22T21:08:17.592 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:17 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:17.592 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:17 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:17.592 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:17 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:17.593 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:17 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:17.593 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:17 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:17.593 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:17 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:17.593 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:17 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:17.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[26394]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:17.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[26394]: osdmap e26: 4 total, 4 up, 4 in 2024-04-22T21:08:17.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:17.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:17.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:17.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:17.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:17.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:17.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:17.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[21867]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T21:08:17.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[21867]: osdmap e26: 4 total, 4 up, 4 in 2024-04-22T21:08:17.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:17.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:17.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:17.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:17.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:17.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:17.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:17 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:18.596 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:18 smithi150 ceph-mon[22676]: Detected new or changed devices on smithi130 2024-04-22T21:08:18.596 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:18 smithi150 ceph-mon[22676]: Adjusting osd_memory_target on smithi130 to 4016M 2024-04-22T21:08:18.596 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:18 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:08:18.596 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:18 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:08:18.596 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:18 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:18 smithi130 ceph-mon[21867]: Detected new or changed devices on smithi130 2024-04-22T21:08:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:18 smithi130 ceph-mon[21867]: Adjusting osd_memory_target on smithi130 to 4016M 2024-04-22T21:08:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:18 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:08:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:18 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:08:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:18 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:18 smithi130 ceph-mon[26394]: Detected new or changed devices on smithi130 2024-04-22T21:08:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:18 smithi130 ceph-mon[26394]: Adjusting osd_memory_target on smithi130 to 4016M 2024-04-22T21:08:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:18 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:08:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:18 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:08:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:18 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:19.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:19 smithi130 ceph-mon[26394]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:19.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:19 smithi130 ceph-mon[26394]: from='client.34211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:08:19.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:19 smithi130 ceph-mon[21867]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:19.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:19 smithi130 ceph-mon[21867]: from='client.34211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:08:19.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:19 smithi150 ceph-mon[22676]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:19.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:19 smithi150 ceph-mon[22676]: from='client.34211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:08:20.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:20 smithi130 ceph-mon[21867]: from='client.? 172.21.15.150:0/3574214462' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "872ec63d-c15e-4680-ab71-e1cf974154cb"}]: dispatch 2024-04-22T21:08:20.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:20 smithi130 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "872ec63d-c15e-4680-ab71-e1cf974154cb"}]: dispatch 2024-04-22T21:08:20.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:20 smithi130 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "872ec63d-c15e-4680-ab71-e1cf974154cb"}]': finished 2024-04-22T21:08:20.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:20 smithi130 ceph-mon[21867]: osdmap e27: 5 total, 4 up, 5 in 2024-04-22T21:08:20.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:20 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:20.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:20 smithi130 ceph-mon[26394]: from='client.? 172.21.15.150:0/3574214462' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "872ec63d-c15e-4680-ab71-e1cf974154cb"}]: dispatch 2024-04-22T21:08:20.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:20 smithi130 ceph-mon[26394]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "872ec63d-c15e-4680-ab71-e1cf974154cb"}]: dispatch 2024-04-22T21:08:20.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:20 smithi130 ceph-mon[26394]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "872ec63d-c15e-4680-ab71-e1cf974154cb"}]': finished 2024-04-22T21:08:20.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:20 smithi130 ceph-mon[26394]: osdmap e27: 5 total, 4 up, 5 in 2024-04-22T21:08:20.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:20 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:20.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:20 smithi150 ceph-mon[22676]: from='client.? 172.21.15.150:0/3574214462' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "872ec63d-c15e-4680-ab71-e1cf974154cb"}]: dispatch 2024-04-22T21:08:20.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:20 smithi150 ceph-mon[22676]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "872ec63d-c15e-4680-ab71-e1cf974154cb"}]: dispatch 2024-04-22T21:08:20.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:20 smithi150 ceph-mon[22676]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "872ec63d-c15e-4680-ab71-e1cf974154cb"}]': finished 2024-04-22T21:08:20.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:20 smithi150 ceph-mon[22676]: osdmap e27: 5 total, 4 up, 5 in 2024-04-22T21:08:20.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:20 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:21.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:21 smithi130 ceph-mon[21867]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:21.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:21 smithi130 ceph-mon[21867]: from='client.? 172.21.15.150:0/1258593228' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:08:21.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:21 smithi130 ceph-mon[26394]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:21.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:21 smithi130 ceph-mon[26394]: from='client.? 172.21.15.150:0/1258593228' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:08:21.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:21 smithi150 ceph-mon[22676]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:21.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:21 smithi150 ceph-mon[22676]: from='client.? 172.21.15.150:0/1258593228' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:08:23.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:23 smithi130 ceph-mon[21867]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:23.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:23 smithi130 ceph-mon[26394]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:23.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:23 smithi150 ceph-mon[22676]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:25.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:25 smithi130 ceph-mon[21867]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:25.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:25 smithi130 ceph-mon[26394]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:25.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:25 smithi150 ceph-mon[22676]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:27.595 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:27 smithi150 ceph-mon[22676]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:27.596 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:27 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T21:08:27.596 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:27 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:27.596 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:27 smithi150 ceph-mon[22676]: Deploying daemon osd.4 on smithi150 2024-04-22T21:08:27.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:27 smithi130 ceph-mon[21867]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:27.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T21:08:27.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:27.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:27 smithi130 ceph-mon[21867]: Deploying daemon osd.4 on smithi150 2024-04-22T21:08:27.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:27 smithi130 ceph-mon[26394]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:27.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T21:08:27.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:27.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:27 smithi130 ceph-mon[26394]: Deploying daemon osd.4 on smithi150 2024-04-22T21:08:29.725 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:29 smithi150 ceph-mon[22676]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:29.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:29 smithi130 ceph-mon[26394]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:29.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:29 smithi130 ceph-mon[21867]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:31.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:31 smithi130 ceph-mon[21867]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:31.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:31 smithi130 ceph-mon[26394]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:31.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:31 smithi150 ceph-mon[22676]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:32.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:32 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:32.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:32 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:32.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:32 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:32.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:32 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:32.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:32 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:32.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:32 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:32.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:32 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:32.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:32 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:32.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:32 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:32.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:32 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:32.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:32 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:32.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:32 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:33.767 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:33 smithi150 ceph-mon[22676]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:33.768 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:33 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:33.768 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:33 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:33.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:33 smithi130 ceph-mon[21867]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:33.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:33 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:33.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:33 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:33.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:33 smithi130 ceph-mon[26394]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:33.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:33 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:33.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:33 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:34.962 INFO:teuthology.orchestra.run.smithi150.stdout:Created osd(s) 4 on host 'smithi150' 2024-04-22T21:08:35.573 DEBUG:teuthology.orchestra.run.smithi150:osd.4> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.4.service 2024-04-22T21:08:35.576 INFO:tasks.cephadm:Deploying osd.5 on smithi150 with /dev/vg_nvme/lv_3... 2024-04-22T21:08:35.576 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-22T21:08:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[21867]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[21867]: from='osd.4 [v2:172.21.15.150:6800/3400040487,v1:172.21.15.150:6801/3400040487]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T21:08:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[21867]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T21:08:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:35.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:35.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[26394]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[26394]: from='osd.4 [v2:172.21.15.150:6800/3400040487,v1:172.21.15.150:6801/3400040487]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T21:08:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[26394]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T21:08:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:35.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:35 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:35.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:35 smithi150 ceph-mon[22676]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:35.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:35 smithi150 ceph-mon[22676]: from='osd.4 [v2:172.21.15.150:6800/3400040487,v1:172.21.15.150:6801/3400040487]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T21:08:35.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:35 smithi150 ceph-mon[22676]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T21:08:35.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:35 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:35.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:35 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:35.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:35 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:35.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:35 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:36.666 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:36 smithi150 ceph-mon[22676]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T21:08:36.666 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:36 smithi150 ceph-mon[22676]: osdmap e28: 5 total, 4 up, 5 in 2024-04-22T21:08:36.666 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:36 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:36.667 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:36 smithi150 ceph-mon[22676]: from='osd.4 [v2:172.21.15.150:6800/3400040487,v1:172.21.15.150:6801/3400040487]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:08:36.667 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:36 smithi150 ceph-mon[22676]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:08:36.667 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:08:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[26102]: 2024-04-22T21:08:36.561+0000 7fad2124d700 -1 osd.4 0 waiting for initial osdmap 2024-04-22T21:08:36.667 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:08:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[26102]: 2024-04-22T21:08:36.566+0000 7fad1dbe8700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:08:36.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:36 smithi130 ceph-mon[26394]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T21:08:36.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:36 smithi130 ceph-mon[26394]: osdmap e28: 5 total, 4 up, 5 in 2024-04-22T21:08:36.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:36 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:36.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:36 smithi130 ceph-mon[26394]: from='osd.4 [v2:172.21.15.150:6800/3400040487,v1:172.21.15.150:6801/3400040487]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:08:36.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:36 smithi130 ceph-mon[26394]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:08:36.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:36 smithi130 ceph-mon[21867]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T21:08:36.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:36 smithi130 ceph-mon[21867]: osdmap e28: 5 total, 4 up, 5 in 2024-04-22T21:08:36.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:36 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:36.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:36 smithi130 ceph-mon[21867]: from='osd.4 [v2:172.21.15.150:6800/3400040487,v1:172.21.15.150:6801/3400040487]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:08:36.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:36 smithi130 ceph-mon[21867]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:08:37.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:37 smithi150 ceph-mon[22676]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:37.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:37 smithi150 ceph-mon[22676]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-22T21:08:37.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:37 smithi150 ceph-mon[22676]: osdmap e29: 5 total, 4 up, 5 in 2024-04-22T21:08:37.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:37 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:37.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:37 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:37 smithi130 ceph-mon[26394]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:37 smithi130 ceph-mon[26394]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-22T21:08:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:37 smithi130 ceph-mon[26394]: osdmap e29: 5 total, 4 up, 5 in 2024-04-22T21:08:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:37 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:37 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:38.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:37 smithi130 ceph-mon[21867]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:38.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:37 smithi130 ceph-mon[21867]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-22T21:08:38.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:37 smithi130 ceph-mon[21867]: osdmap e29: 5 total, 4 up, 5 in 2024-04-22T21:08:38.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:37 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:38.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:37 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:38.812 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:38 smithi150 ceph-mon[22676]: purged_snaps scrub starts 2024-04-22T21:08:38.812 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:38 smithi150 ceph-mon[22676]: purged_snaps scrub ok 2024-04-22T21:08:38.812 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:38.812 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:38 smithi150 ceph-mon[22676]: osd.4 [v2:172.21.15.150:6800/3400040487,v1:172.21.15.150:6801/3400040487] boot 2024-04-22T21:08:38.812 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:38 smithi150 ceph-mon[22676]: osdmap e30: 5 total, 5 up, 5 in 2024-04-22T21:08:38.812 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:38.812 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:38.812 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:38.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:38.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:38 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:39.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[26394]: purged_snaps scrub starts 2024-04-22T21:08:39.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[26394]: purged_snaps scrub ok 2024-04-22T21:08:39.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:39.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[26394]: osd.4 [v2:172.21.15.150:6800/3400040487,v1:172.21.15.150:6801/3400040487] boot 2024-04-22T21:08:39.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[26394]: osdmap e30: 5 total, 5 up, 5 in 2024-04-22T21:08:39.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:39.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:39.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:39.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:39.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:39.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[21867]: purged_snaps scrub starts 2024-04-22T21:08:39.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[21867]: purged_snaps scrub ok 2024-04-22T21:08:39.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:39.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[21867]: osd.4 [v2:172.21.15.150:6800/3400040487,v1:172.21.15.150:6801/3400040487] boot 2024-04-22T21:08:39.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[21867]: osdmap e30: 5 total, 5 up, 5 in 2024-04-22T21:08:39.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:08:39.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:39.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:08:39.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:39.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:38 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:39.543 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-22T21:08:39.543 INFO:teuthology.orchestra.run.smithi150.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-22T21:08:39.543 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10+0 records in 2024-04-22T21:08:39.543 INFO:teuthology.orchestra.run.smithi150.stderr:10+0 records out 2024-04-22T21:08:39.543 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0121018 s, 866 MB/s 2024-04-22T21:08:39.543 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping successful for: 2024-04-22T21:08:39.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:39 smithi150 ceph-mon[22676]: pgmap v108: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:39.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:39 smithi150 ceph-mon[22676]: Detected new or changed devices on smithi150 2024-04-22T21:08:39.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:39 smithi150 ceph-mon[22676]: Adjusting osd_memory_target on smithi150 to 17088M 2024-04-22T21:08:39.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:39 smithi150 ceph-mon[22676]: osdmap e31: 5 total, 5 up, 5 in 2024-04-22T21:08:40.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:39 smithi130 ceph-mon[21867]: pgmap v108: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:40.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:39 smithi130 ceph-mon[21867]: Detected new or changed devices on smithi150 2024-04-22T21:08:40.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:39 smithi130 ceph-mon[21867]: Adjusting osd_memory_target on smithi150 to 17088M 2024-04-22T21:08:40.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:39 smithi130 ceph-mon[21867]: osdmap e31: 5 total, 5 up, 5 in 2024-04-22T21:08:40.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:39 smithi130 ceph-mon[26394]: pgmap v108: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T21:08:40.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:39 smithi130 ceph-mon[26394]: Detected new or changed devices on smithi150 2024-04-22T21:08:40.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:39 smithi130 ceph-mon[26394]: Adjusting osd_memory_target on smithi150 to 17088M 2024-04-22T21:08:40.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:39 smithi130 ceph-mon[26394]: osdmap e31: 5 total, 5 up, 5 in 2024-04-22T21:08:40.176 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi150:vg_nvme/lv_3 2024-04-22T21:08:40.796 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:40 smithi150 ceph-mon[22676]: osdmap e32: 5 total, 5 up, 5 in 2024-04-22T21:08:41.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:40 smithi130 ceph-mon[26394]: osdmap e32: 5 total, 5 up, 5 in 2024-04-22T21:08:41.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:40 smithi130 ceph-mon[21867]: osdmap e32: 5 total, 5 up, 5 in 2024-04-22T21:08:41.676 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:41 smithi150 ceph-mon[22676]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:41.677 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:41 smithi150 ceph-mon[22676]: osdmap e33: 5 total, 5 up, 5 in 2024-04-22T21:08:42.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:41 smithi130 ceph-mon[26394]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:42.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:41 smithi130 ceph-mon[26394]: osdmap e33: 5 total, 5 up, 5 in 2024-04-22T21:08:42.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:41 smithi130 ceph-mon[21867]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:42.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:41 smithi130 ceph-mon[21867]: osdmap e33: 5 total, 5 up, 5 in 2024-04-22T21:08:42.684 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:42 smithi150 ceph-mon[22676]: osdmap e34: 5 total, 5 up, 5 in 2024-04-22T21:08:42.826 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:42 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:08:42.826 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:42 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:08:42.826 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:42 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:08:42.826 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:42 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:08:42.826 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:42 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:43.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:42 smithi130 ceph-mon[26394]: osdmap e34: 5 total, 5 up, 5 in 2024-04-22T21:08:43.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:42 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:08:43.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:42 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:08:43.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:42 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:08:43.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:42 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:08:43.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:42 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:43.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:42 smithi130 ceph-mon[21867]: osdmap e34: 5 total, 5 up, 5 in 2024-04-22T21:08:43.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:08:43.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:08:43.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:08:43.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:08:43.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:43.669 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:43 smithi150 ceph-mon[22676]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:43.669 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:43 smithi150 ceph-mon[22676]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:08:44.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:43 smithi130 ceph-mon[26394]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:44.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:43 smithi130 ceph-mon[26394]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:08:44.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:43 smithi130 ceph-mon[21867]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:44.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:43 smithi130 ceph-mon[21867]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:08:45.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:44 smithi130 ceph-mon[26394]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-04-22T21:08:45.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:44 smithi130 ceph-mon[26394]: from='client.? 172.21.15.150:0/1097535517' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67066d12-b377-4cc6-93f0-ce22e7fb1de2"}]: dispatch 2024-04-22T21:08:45.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:44 smithi130 ceph-mon[26394]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67066d12-b377-4cc6-93f0-ce22e7fb1de2"}]: dispatch 2024-04-22T21:08:45.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:44 smithi130 ceph-mon[26394]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "67066d12-b377-4cc6-93f0-ce22e7fb1de2"}]': finished 2024-04-22T21:08:45.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:44 smithi130 ceph-mon[26394]: osdmap e35: 6 total, 5 up, 6 in 2024-04-22T21:08:45.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:44 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:08:45.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:44 smithi130 ceph-mon[21867]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-04-22T21:08:45.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:44 smithi130 ceph-mon[21867]: from='client.? 172.21.15.150:0/1097535517' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67066d12-b377-4cc6-93f0-ce22e7fb1de2"}]: dispatch 2024-04-22T21:08:45.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:44 smithi130 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67066d12-b377-4cc6-93f0-ce22e7fb1de2"}]: dispatch 2024-04-22T21:08:45.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:44 smithi130 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "67066d12-b377-4cc6-93f0-ce22e7fb1de2"}]': finished 2024-04-22T21:08:45.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:44 smithi130 ceph-mon[21867]: osdmap e35: 6 total, 5 up, 6 in 2024-04-22T21:08:45.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:44 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:08:45.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:44 smithi150 ceph-mon[22676]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-04-22T21:08:45.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:44 smithi150 ceph-mon[22676]: from='client.? 172.21.15.150:0/1097535517' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67066d12-b377-4cc6-93f0-ce22e7fb1de2"}]: dispatch 2024-04-22T21:08:45.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:44 smithi150 ceph-mon[22676]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67066d12-b377-4cc6-93f0-ce22e7fb1de2"}]: dispatch 2024-04-22T21:08:45.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:44 smithi150 ceph-mon[22676]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "67066d12-b377-4cc6-93f0-ce22e7fb1de2"}]': finished 2024-04-22T21:08:45.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:44 smithi150 ceph-mon[22676]: osdmap e35: 6 total, 5 up, 6 in 2024-04-22T21:08:45.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:44 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:08:46.226 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:45 smithi130 ceph-mon[21867]: from='client.? 172.21.15.150:0/1887624653' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:08:46.226 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:45 smithi130 ceph-mon[26394]: from='client.? 172.21.15.150:0/1887624653' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:08:46.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:45 smithi150 ceph-mon[22676]: from='client.? 172.21.15.150:0/1887624653' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:08:47.222 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:46 smithi150 ceph-mon[22676]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T21:08:47.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:46 smithi130 ceph-mon[26394]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T21:08:47.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:46 smithi130 ceph-mon[21867]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T21:08:49.281 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:48 smithi150 ceph-mon[22676]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-22T21:08:49.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:48 smithi130 ceph-mon[26394]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-22T21:08:49.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:48 smithi130 ceph-mon[21867]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-22T21:08:51.224 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:50 smithi150 ceph-mon[22676]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-22T21:08:51.224 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:50 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T21:08:51.224 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:50 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:51.224 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:50 smithi150 ceph-mon[22676]: Deploying daemon osd.5 on smithi150 2024-04-22T21:08:51.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:50 smithi130 ceph-mon[26394]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-22T21:08:51.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:50 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T21:08:51.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:50 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:51.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:50 smithi130 ceph-mon[26394]: Deploying daemon osd.5 on smithi150 2024-04-22T21:08:51.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:50 smithi130 ceph-mon[21867]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-22T21:08:51.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:50 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T21:08:51.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:50 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:51.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:50 smithi130 ceph-mon[21867]: Deploying daemon osd.5 on smithi150 2024-04-22T21:08:53.235 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:52 smithi150 ceph-mon[22676]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T21:08:53.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:52 smithi130 ceph-mon[21867]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T21:08:53.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:52 smithi130 ceph-mon[26394]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T21:08:55.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:54 smithi150 ceph-mon[22676]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:55.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:54 smithi130 ceph-mon[21867]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:55.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:54 smithi130 ceph-mon[26394]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:56.945 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:56 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:56.945 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:56 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:56.945 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:56 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:56.945 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:56 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:57.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:56 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:57.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:56 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:57.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:56 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:57.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:56 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:57.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:56 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:08:57.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:56 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:57.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:56 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:08:57.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:56 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:08:57.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:57 smithi130 ceph-mon[21867]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:57.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:57 smithi130 ceph-mon[26394]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:58.021 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:57 smithi150 ceph-mon[22676]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:58.347 INFO:teuthology.orchestra.run.smithi150.stdout:Created osd(s) 5 on host 'smithi150' 2024-04-22T21:08:59.302 DEBUG:teuthology.orchestra.run.smithi150:osd.5> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.5.service 2024-04-22T21:08:59.304 INFO:tasks.cephadm:Deploying osd.6 on smithi150 with /dev/vg_nvme/lv_2... 2024-04-22T21:08:59.305 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-22T21:08:59.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:59 smithi150 ceph-mon[22676]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:59.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:59 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:59.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:59 smithi150 ceph-mon[22676]: from='osd.5 [v2:172.21.15.150:6808/3183024588,v1:172.21.15.150:6809/3183024588]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T21:08:59.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:08:59 smithi150 ceph-mon[22676]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T21:08:59.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:59 smithi130 ceph-mon[21867]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:59.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:59 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:59.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:59 smithi130 ceph-mon[21867]: from='osd.5 [v2:172.21.15.150:6808/3183024588,v1:172.21.15.150:6809/3183024588]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T21:08:59.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:08:59 smithi130 ceph-mon[21867]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T21:08:59.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:59 smithi130 ceph-mon[26394]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:08:59.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:59 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:08:59.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:59 smithi130 ceph-mon[26394]: from='osd.5 [v2:172.21.15.150:6808/3183024588,v1:172.21.15.150:6809/3183024588]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T21:08:59.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:08:59 smithi130 ceph-mon[26394]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T21:09:00.638 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:00 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:00.638 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:00 smithi150 ceph-mon[22676]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T21:09:00.638 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:00 smithi150 ceph-mon[22676]: osdmap e36: 6 total, 5 up, 6 in 2024-04-22T21:09:00.638 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:00 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:09:00.638 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:00 smithi150 ceph-mon[22676]: from='osd.5 [v2:172.21.15.150:6808/3183024588,v1:172.21.15.150:6809/3183024588]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:00.638 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:00 smithi150 ceph-mon[22676]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:00.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:00 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:00.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:00 smithi130 ceph-mon[21867]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T21:09:00.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:00 smithi130 ceph-mon[21867]: osdmap e36: 6 total, 5 up, 6 in 2024-04-22T21:09:00.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:00 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:09:00.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:00 smithi130 ceph-mon[21867]: from='osd.5 [v2:172.21.15.150:6808/3183024588,v1:172.21.15.150:6809/3183024588]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:00.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:00 smithi130 ceph-mon[21867]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:00.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:00 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:00.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:00 smithi130 ceph-mon[26394]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T21:09:00.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:00 smithi130 ceph-mon[26394]: osdmap e36: 6 total, 5 up, 6 in 2024-04-22T21:09:00.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:00 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:09:00.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:00 smithi130 ceph-mon[26394]: from='osd.5 [v2:172.21.15.150:6808/3183024588,v1:172.21.15.150:6809/3183024588]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:00.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:00 smithi130 ceph-mon[26394]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:00.973 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:09:00 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[28986]: 2024-04-22T21:09:00.641+0000 7f25797ab700 -1 osd.5 0 waiting for initial osdmap 2024-04-22T21:09:00.973 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:09:00 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[28986]: 2024-04-22T21:09:00.646+0000 7f257213e700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:09:01.433 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-22T21:09:01.433 INFO:teuthology.orchestra.run.smithi150.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-22T21:09:01.433 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10+0 records in 2024-04-22T21:09:01.433 INFO:teuthology.orchestra.run.smithi150.stderr:10+0 records out 2024-04-22T21:09:01.433 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0122262 s, 858 MB/s 2024-04-22T21:09:01.433 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping successful for: 2024-04-22T21:09:01.896 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:01 smithi150 ceph-mon[22676]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:09:01.896 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:01 smithi150 ceph-mon[22676]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-22T21:09:01.896 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:01 smithi150 ceph-mon[22676]: osdmap e37: 6 total, 5 up, 6 in 2024-04-22T21:09:01.896 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:01 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:09:01.896 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:01 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:09:02.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:01 smithi130 ceph-mon[21867]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:09:02.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:01 smithi130 ceph-mon[21867]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-22T21:09:02.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:01 smithi130 ceph-mon[21867]: osdmap e37: 6 total, 5 up, 6 in 2024-04-22T21:09:02.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:01 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:09:02.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:01 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:09:02.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:01 smithi130 ceph-mon[26394]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T21:09:02.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:01 smithi130 ceph-mon[26394]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-22T21:09:02.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:01 smithi130 ceph-mon[26394]: osdmap e37: 6 total, 5 up, 6 in 2024-04-22T21:09:02.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:01 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:09:02.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:01 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:09:02.591 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi150:vg_nvme/lv_2 2024-04-22T21:09:02.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:02 smithi150 ceph-mon[22676]: purged_snaps scrub starts 2024-04-22T21:09:02.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:02 smithi150 ceph-mon[22676]: purged_snaps scrub ok 2024-04-22T21:09:02.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:02 smithi150 ceph-mon[22676]: osd.5 [v2:172.21.15.150:6808/3183024588,v1:172.21.15.150:6809/3183024588] boot 2024-04-22T21:09:02.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:02 smithi150 ceph-mon[22676]: osdmap e38: 6 total, 6 up, 6 in 2024-04-22T21:09:02.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:02 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:09:03.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:02 smithi130 ceph-mon[21867]: purged_snaps scrub starts 2024-04-22T21:09:03.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:02 smithi130 ceph-mon[21867]: purged_snaps scrub ok 2024-04-22T21:09:03.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:02 smithi130 ceph-mon[21867]: osd.5 [v2:172.21.15.150:6808/3183024588,v1:172.21.15.150:6809/3183024588] boot 2024-04-22T21:09:03.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:02 smithi130 ceph-mon[21867]: osdmap e38: 6 total, 6 up, 6 in 2024-04-22T21:09:03.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:02 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:09:03.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:02 smithi130 ceph-mon[26394]: purged_snaps scrub starts 2024-04-22T21:09:03.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:02 smithi130 ceph-mon[26394]: purged_snaps scrub ok 2024-04-22T21:09:03.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:02 smithi130 ceph-mon[26394]: osd.5 [v2:172.21.15.150:6808/3183024588,v1:172.21.15.150:6809/3183024588] boot 2024-04-22T21:09:03.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:02 smithi130 ceph-mon[26394]: osdmap e38: 6 total, 6 up, 6 in 2024-04-22T21:09:03.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:02 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:09:03.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:03 smithi150 ceph-mon[22676]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:03.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:03 smithi150 ceph-mon[22676]: osdmap e39: 6 total, 6 up, 6 in 2024-04-22T21:09:04.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:03 smithi130 ceph-mon[21867]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:04.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:03 smithi130 ceph-mon[21867]: osdmap e39: 6 total, 6 up, 6 in 2024-04-22T21:09:04.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:03 smithi130 ceph-mon[26394]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:04.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:03 smithi130 ceph-mon[26394]: osdmap e39: 6 total, 6 up, 6 in 2024-04-22T21:09:04.773 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:04 smithi150 ceph-mon[22676]: osdmap e40: 6 total, 6 up, 6 in 2024-04-22T21:09:04.773 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:04 smithi150 ceph-mon[22676]: osdmap e41: 6 total, 6 up, 6 in 2024-04-22T21:09:05.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:04 smithi130 ceph-mon[21867]: osdmap e40: 6 total, 6 up, 6 in 2024-04-22T21:09:05.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:04 smithi130 ceph-mon[21867]: osdmap e41: 6 total, 6 up, 6 in 2024-04-22T21:09:05.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:04 smithi130 ceph-mon[26394]: osdmap e40: 6 total, 6 up, 6 in 2024-04-22T21:09:05.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:04 smithi130 ceph-mon[26394]: osdmap e41: 6 total, 6 up, 6 in 2024-04-22T21:09:05.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:05 smithi150 ceph-mon[22676]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:05.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:05 smithi150 ceph-mon[22676]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T21:09:05.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:05 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:05.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:05 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:05.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:05 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:05.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:05 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:05.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:05 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:05.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:05 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:05.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:05 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:05.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:05 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:05.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:05 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:05.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:05 smithi150 ceph-mon[22676]: osdmap e42: 6 total, 6 up, 6 in 2024-04-22T21:09:06.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[21867]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:06.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[21867]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T21:09:06.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:06.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:06.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:06.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:06.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:06.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:06.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:06.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:06.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:06.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[21867]: osdmap e42: 6 total, 6 up, 6 in 2024-04-22T21:09:06.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[26394]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:06.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[26394]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T21:09:06.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:06.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:06.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:06.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:06.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:06.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:06.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:06.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:06.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:06.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:05 smithi130 ceph-mon[26394]: osdmap e42: 6 total, 6 up, 6 in 2024-04-22T21:09:06.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:06 smithi150 ceph-mon[22676]: Detected new or changed devices on smithi150 2024-04-22T21:09:06.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:06 smithi150 ceph-mon[22676]: Adjusting osd_memory_target on smithi150 to 8544M 2024-04-22T21:09:06.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:06 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:09:06.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:06 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:09:06.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:06 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:07.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:06 smithi130 ceph-mon[21867]: Detected new or changed devices on smithi150 2024-04-22T21:09:07.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:06 smithi130 ceph-mon[21867]: Adjusting osd_memory_target on smithi150 to 8544M 2024-04-22T21:09:07.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:06 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:09:07.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:06 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:09:07.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:06 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:07.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:06 smithi130 ceph-mon[26394]: Detected new or changed devices on smithi150 2024-04-22T21:09:07.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:06 smithi130 ceph-mon[26394]: Adjusting osd_memory_target on smithi150 to 8544M 2024-04-22T21:09:07.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:06 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:09:07.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:06 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:09:07.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:06 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:07.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:07 smithi150 ceph-mon[22676]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:07.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:07 smithi150 ceph-mon[22676]: from='client.34265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:09:08.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:07 smithi130 ceph-mon[21867]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:08.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:07 smithi130 ceph-mon[21867]: from='client.34265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:09:08.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:07 smithi130 ceph-mon[26394]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:08.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:07 smithi130 ceph-mon[26394]: from='client.34265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:09:09.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:08 smithi130 ceph-mon[21867]: from='client.? 172.21.15.150:0/58525638' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f18c163-a1a4-4016-9848-1d43db8a0c52"}]: dispatch 2024-04-22T21:09:09.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:08 smithi130 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f18c163-a1a4-4016-9848-1d43db8a0c52"}]: dispatch 2024-04-22T21:09:09.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:08 smithi130 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f18c163-a1a4-4016-9848-1d43db8a0c52"}]': finished 2024-04-22T21:09:09.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:08 smithi130 ceph-mon[21867]: osdmap e43: 7 total, 6 up, 7 in 2024-04-22T21:09:09.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:08 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:09.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:08 smithi130 ceph-mon[26394]: from='client.? 172.21.15.150:0/58525638' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f18c163-a1a4-4016-9848-1d43db8a0c52"}]: dispatch 2024-04-22T21:09:09.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:08 smithi130 ceph-mon[26394]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f18c163-a1a4-4016-9848-1d43db8a0c52"}]: dispatch 2024-04-22T21:09:09.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:08 smithi130 ceph-mon[26394]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f18c163-a1a4-4016-9848-1d43db8a0c52"}]': finished 2024-04-22T21:09:09.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:08 smithi130 ceph-mon[26394]: osdmap e43: 7 total, 6 up, 7 in 2024-04-22T21:09:09.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:08 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:09.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:08 smithi150 ceph-mon[22676]: from='client.? 172.21.15.150:0/58525638' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f18c163-a1a4-4016-9848-1d43db8a0c52"}]: dispatch 2024-04-22T21:09:09.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:08 smithi150 ceph-mon[22676]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f18c163-a1a4-4016-9848-1d43db8a0c52"}]: dispatch 2024-04-22T21:09:09.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:08 smithi150 ceph-mon[22676]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f18c163-a1a4-4016-9848-1d43db8a0c52"}]': finished 2024-04-22T21:09:09.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:08 smithi150 ceph-mon[22676]: osdmap e43: 7 total, 6 up, 7 in 2024-04-22T21:09:09.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:08 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:10.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:09 smithi130 ceph-mon[21867]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:10.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:09 smithi130 ceph-mon[21867]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T21:09:10.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:09 smithi130 ceph-mon[21867]: Cluster is now healthy 2024-04-22T21:09:10.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:09 smithi130 ceph-mon[21867]: from='client.? 172.21.15.150:0/3759914500' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:09:10.062 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:09 smithi130 ceph-mon[26394]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:10.062 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:09 smithi130 ceph-mon[26394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T21:09:10.062 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:09 smithi130 ceph-mon[26394]: Cluster is now healthy 2024-04-22T21:09:10.062 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:09 smithi130 ceph-mon[26394]: from='client.? 172.21.15.150:0/3759914500' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:09:10.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:09 smithi150 ceph-mon[22676]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:10.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:09 smithi150 ceph-mon[22676]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T21:09:10.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:09 smithi150 ceph-mon[22676]: Cluster is now healthy 2024-04-22T21:09:10.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:09 smithi150 ceph-mon[22676]: from='client.? 172.21.15.150:0/3759914500' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:09:11.771 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:11 smithi150 ceph-mon[22676]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T21:09:12.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:11 smithi130 ceph-mon[21867]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T21:09:12.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:11 smithi130 ceph-mon[26394]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T21:09:13.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:13 smithi150 ceph-mon[22676]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-22T21:09:14.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:13 smithi130 ceph-mon[21867]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-22T21:09:14.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:13 smithi130 ceph-mon[26394]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-22T21:09:16.023 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:15 smithi150 ceph-mon[22676]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-22T21:09:16.023 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:15 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T21:09:16.023 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:15 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:16.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:15 smithi150 ceph-mon[22676]: Deploying daemon osd.6 on smithi150 2024-04-22T21:09:16.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:15 smithi130 ceph-mon[21867]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-22T21:09:16.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:15 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T21:09:16.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:15 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:16.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:15 smithi130 ceph-mon[21867]: Deploying daemon osd.6 on smithi150 2024-04-22T21:09:16.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:15 smithi130 ceph-mon[26394]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-22T21:09:16.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:15 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T21:09:16.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:15 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:16.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:15 smithi130 ceph-mon[26394]: Deploying daemon osd.6 on smithi150 2024-04-22T21:09:17.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:17 smithi150 ceph-mon[22676]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T21:09:18.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:17 smithi130 ceph-mon[21867]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T21:09:18.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:17 smithi130 ceph-mon[26394]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T21:09:19.985 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:19 smithi130 ceph-mon[21867]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:19.985 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:19 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:19.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:19 smithi130 ceph-mon[26394]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:19.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:19 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:20.041 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:19 smithi150 ceph-mon[22676]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:20.041 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:19 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:20.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:20 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:20.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:20 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:20.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:20 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:21.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:20 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:21.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:20 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:21.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:20 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:21.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:20 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:21.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:20 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:21.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:20 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:21.766 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:21 smithi150 ceph-mon[22676]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:22.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:21 smithi130 ceph-mon[21867]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:22.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:21 smithi130 ceph-mon[26394]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:23.248 INFO:teuthology.orchestra.run.smithi150.stdout:Created osd(s) 6 on host 'smithi150' 2024-04-22T21:09:23.559 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:23 smithi150 ceph-mon[22676]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:23.559 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:23 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:23.559 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:23 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:23.560 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:23 smithi150 ceph-mon[22676]: from='osd.6 [v2:172.21.15.150:6816/3361358828,v1:172.21.15.150:6817/3361358828]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T21:09:23.560 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:23 smithi150 ceph-mon[22676]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T21:09:23.560 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:23 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:23.560 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:23 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:23.560 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:23 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:23.560 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:23 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:23.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[21867]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:23.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:23.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:23.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[21867]: from='osd.6 [v2:172.21.15.150:6816/3361358828,v1:172.21.15.150:6817/3361358828]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T21:09:23.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[21867]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T21:09:23.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:23.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:23.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:23.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:23.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[26394]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:23.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:23.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:23.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[26394]: from='osd.6 [v2:172.21.15.150:6816/3361358828,v1:172.21.15.150:6817/3361358828]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T21:09:23.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[26394]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T21:09:23.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:23.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:23.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:23.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:23 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:23.875 DEBUG:teuthology.orchestra.run.smithi150:osd.6> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.6.service 2024-04-22T21:09:23.879 INFO:tasks.cephadm:Deploying osd.7 on smithi150 with /dev/vg_nvme/lv_1... 2024-04-22T21:09:23.879 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-22T21:09:24.952 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:24 smithi150 ceph-mon[22676]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T21:09:24.952 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:24 smithi150 ceph-mon[22676]: osdmap e44: 7 total, 6 up, 7 in 2024-04-22T21:09:24.952 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:24 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:24.952 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:24 smithi150 ceph-mon[22676]: from='osd.6 [v2:172.21.15.150:6816/3361358828,v1:172.21.15.150:6817/3361358828]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:24.952 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:24 smithi150 ceph-mon[22676]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:24.952 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:09:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[31883]: 2024-04-22T21:09:24.709+0000 7f040de01700 -1 osd.6 0 waiting for initial osdmap 2024-04-22T21:09:24.952 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:09:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[31883]: 2024-04-22T21:09:24.715+0000 7f0408798700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:09:25.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:24 smithi130 ceph-mon[21867]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T21:09:25.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:24 smithi130 ceph-mon[21867]: osdmap e44: 7 total, 6 up, 7 in 2024-04-22T21:09:25.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:24 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:25.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:24 smithi130 ceph-mon[21867]: from='osd.6 [v2:172.21.15.150:6816/3361358828,v1:172.21.15.150:6817/3361358828]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:25.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:24 smithi130 ceph-mon[21867]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:25.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:24 smithi130 ceph-mon[26394]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T21:09:25.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:24 smithi130 ceph-mon[26394]: osdmap e44: 7 total, 6 up, 7 in 2024-04-22T21:09:25.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:24 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:25.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:24 smithi130 ceph-mon[26394]: from='osd.6 [v2:172.21.15.150:6816/3361358828,v1:172.21.15.150:6817/3361358828]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:25.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:24 smithi130 ceph-mon[26394]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:25.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:25 smithi150 ceph-mon[22676]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:25.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:25 smithi150 ceph-mon[22676]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-22T21:09:25.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:25 smithi150 ceph-mon[22676]: osdmap e45: 7 total, 6 up, 7 in 2024-04-22T21:09:25.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:25 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:25.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:25 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:25.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:25 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:26.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:25 smithi130 ceph-mon[26394]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:26.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:25 smithi130 ceph-mon[26394]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-22T21:09:26.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:25 smithi130 ceph-mon[26394]: osdmap e45: 7 total, 6 up, 7 in 2024-04-22T21:09:26.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:25 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:26.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:25 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:26.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:25 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:26.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:25 smithi130 ceph-mon[21867]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:26.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:25 smithi130 ceph-mon[21867]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-22T21:09:26.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:25 smithi130 ceph-mon[21867]: osdmap e45: 7 total, 6 up, 7 in 2024-04-22T21:09:26.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:25 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:26.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:25 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:26.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:25 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:26.771 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:26 smithi150 ceph-mon[22676]: purged_snaps scrub starts 2024-04-22T21:09:26.772 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:26 smithi150 ceph-mon[22676]: purged_snaps scrub ok 2024-04-22T21:09:26.772 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:26 smithi150 ceph-mon[22676]: osd.6 [v2:172.21.15.150:6816/3361358828,v1:172.21.15.150:6817/3361358828] boot 2024-04-22T21:09:26.772 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:26 smithi150 ceph-mon[22676]: osdmap e46: 7 total, 7 up, 7 in 2024-04-22T21:09:26.772 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:26 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:27.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:26 smithi130 ceph-mon[21867]: purged_snaps scrub starts 2024-04-22T21:09:27.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:26 smithi130 ceph-mon[21867]: purged_snaps scrub ok 2024-04-22T21:09:27.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:26 smithi130 ceph-mon[21867]: osd.6 [v2:172.21.15.150:6816/3361358828,v1:172.21.15.150:6817/3361358828] boot 2024-04-22T21:09:27.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:26 smithi130 ceph-mon[21867]: osdmap e46: 7 total, 7 up, 7 in 2024-04-22T21:09:27.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:26 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:27.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:26 smithi130 ceph-mon[26394]: purged_snaps scrub starts 2024-04-22T21:09:27.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:26 smithi130 ceph-mon[26394]: purged_snaps scrub ok 2024-04-22T21:09:27.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:26 smithi130 ceph-mon[26394]: osd.6 [v2:172.21.15.150:6816/3361358828,v1:172.21.15.150:6817/3361358828] boot 2024-04-22T21:09:27.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:26 smithi130 ceph-mon[26394]: osdmap e46: 7 total, 7 up, 7 in 2024-04-22T21:09:27.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:26 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:09:27.430 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-22T21:09:27.430 INFO:teuthology.orchestra.run.smithi150.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-22T21:09:27.430 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10+0 records in 2024-04-22T21:09:27.430 INFO:teuthology.orchestra.run.smithi150.stderr:10+0 records out 2024-04-22T21:09:27.430 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.013462 s, 779 MB/s 2024-04-22T21:09:27.430 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping successful for: 2024-04-22T21:09:28.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[26394]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:28.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[26394]: osdmap e47: 7 total, 7 up, 7 in 2024-04-22T21:09:28.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:28.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:28.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:28.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:28.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:28.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:28.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[21867]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:28.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[21867]: osdmap e47: 7 total, 7 up, 7 in 2024-04-22T21:09:28.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:28.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:28.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:28.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:28.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:28.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:27 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:28.072 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:27 smithi150 ceph-mon[22676]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T21:09:28.072 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:27 smithi150 ceph-mon[22676]: osdmap e47: 7 total, 7 up, 7 in 2024-04-22T21:09:28.073 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:27 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:28.073 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:27 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:28.073 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:27 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:28.073 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:27 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:28.073 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:27 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:28.073 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:27 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:28.219 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi150:vg_nvme/lv_1 2024-04-22T21:09:28.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:28 smithi150 ceph-mon[22676]: Detected new or changed devices on smithi150 2024-04-22T21:09:28.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:28 smithi150 ceph-mon[22676]: Adjusting osd_memory_target on smithi150 to 5696M 2024-04-22T21:09:28.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:28 smithi150 ceph-mon[22676]: osdmap e48: 7 total, 7 up, 7 in 2024-04-22T21:09:29.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:28 smithi130 ceph-mon[26394]: Detected new or changed devices on smithi150 2024-04-22T21:09:29.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:28 smithi130 ceph-mon[26394]: Adjusting osd_memory_target on smithi150 to 5696M 2024-04-22T21:09:29.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:28 smithi130 ceph-mon[26394]: osdmap e48: 7 total, 7 up, 7 in 2024-04-22T21:09:29.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:28 smithi130 ceph-mon[21867]: Detected new or changed devices on smithi150 2024-04-22T21:09:29.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:28 smithi130 ceph-mon[21867]: Adjusting osd_memory_target on smithi150 to 5696M 2024-04-22T21:09:29.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:28 smithi130 ceph-mon[21867]: osdmap e48: 7 total, 7 up, 7 in 2024-04-22T21:09:29.877 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:29 smithi150 ceph-mon[22676]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:29.878 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:29 smithi150 ceph-mon[22676]: osdmap e49: 7 total, 7 up, 7 in 2024-04-22T21:09:30.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:29 smithi130 ceph-mon[26394]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:30.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:29 smithi130 ceph-mon[26394]: osdmap e49: 7 total, 7 up, 7 in 2024-04-22T21:09:30.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:29 smithi130 ceph-mon[21867]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:30.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:29 smithi130 ceph-mon[21867]: osdmap e49: 7 total, 7 up, 7 in 2024-04-22T21:09:30.873 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:30 smithi150 ceph-mon[22676]: osdmap e50: 7 total, 7 up, 7 in 2024-04-22T21:09:30.873 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:30 smithi150 ceph-mon[22676]: pgmap v154: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-22T21:09:30.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:30 smithi150 ceph-mon[22676]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:09:30.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:30 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:09:30.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:30 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:09:30.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:30 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:31.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:30 smithi130 ceph-mon[21867]: osdmap e50: 7 total, 7 up, 7 in 2024-04-22T21:09:31.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:30 smithi130 ceph-mon[21867]: pgmap v154: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-22T21:09:31.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:30 smithi130 ceph-mon[21867]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:09:31.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:30 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:09:31.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:30 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:09:31.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:30 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:31.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:30 smithi130 ceph-mon[26394]: osdmap e50: 7 total, 7 up, 7 in 2024-04-22T21:09:31.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:30 smithi130 ceph-mon[26394]: pgmap v154: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-22T21:09:31.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:30 smithi130 ceph-mon[26394]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:09:31.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:30 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T21:09:31.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:30 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T21:09:31.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:30 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:33.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:32 smithi130 ceph-mon[26394]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-22T21:09:33.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:32 smithi130 ceph-mon[26394]: from='client.? 172.21.15.150:0/152492250' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f45d779f-52bd-4994-b35d-61601dda32ba"}]: dispatch 2024-04-22T21:09:33.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:32 smithi130 ceph-mon[26394]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f45d779f-52bd-4994-b35d-61601dda32ba"}]: dispatch 2024-04-22T21:09:33.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:32 smithi130 ceph-mon[26394]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f45d779f-52bd-4994-b35d-61601dda32ba"}]': finished 2024-04-22T21:09:33.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:32 smithi130 ceph-mon[26394]: osdmap e51: 8 total, 7 up, 8 in 2024-04-22T21:09:33.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:32 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:33.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:32 smithi130 ceph-mon[21867]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-22T21:09:33.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:32 smithi130 ceph-mon[21867]: from='client.? 172.21.15.150:0/152492250' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f45d779f-52bd-4994-b35d-61601dda32ba"}]: dispatch 2024-04-22T21:09:33.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:32 smithi130 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f45d779f-52bd-4994-b35d-61601dda32ba"}]: dispatch 2024-04-22T21:09:33.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:32 smithi130 ceph-mon[21867]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f45d779f-52bd-4994-b35d-61601dda32ba"}]': finished 2024-04-22T21:09:33.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:32 smithi130 ceph-mon[21867]: osdmap e51: 8 total, 7 up, 8 in 2024-04-22T21:09:33.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:32 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:33.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:32 smithi150 ceph-mon[22676]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-22T21:09:33.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:32 smithi150 ceph-mon[22676]: from='client.? 172.21.15.150:0/152492250' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f45d779f-52bd-4994-b35d-61601dda32ba"}]: dispatch 2024-04-22T21:09:33.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:32 smithi150 ceph-mon[22676]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f45d779f-52bd-4994-b35d-61601dda32ba"}]: dispatch 2024-04-22T21:09:33.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:32 smithi150 ceph-mon[22676]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f45d779f-52bd-4994-b35d-61601dda32ba"}]': finished 2024-04-22T21:09:33.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:32 smithi150 ceph-mon[22676]: osdmap e51: 8 total, 7 up, 8 in 2024-04-22T21:09:33.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:32 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:34.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:33 smithi130 ceph-mon[26394]: from='client.? 172.21.15.150:0/2199042067' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:09:34.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:33 smithi130 ceph-mon[21867]: from='client.? 172.21.15.150:0/2199042067' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:09:34.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:33 smithi150 ceph-mon[22676]: from='client.? 172.21.15.150:0/2199042067' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T21:09:35.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:34 smithi130 ceph-mon[26394]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T21:09:35.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:34 smithi130 ceph-mon[21867]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T21:09:35.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:34 smithi150 ceph-mon[22676]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T21:09:37.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:36 smithi130 ceph-mon[21867]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:37.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:36 smithi130 ceph-mon[26394]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:37.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:36 smithi150 ceph-mon[22676]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:39.203 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:38 smithi150 ceph-mon[22676]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:39.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:38 smithi130 ceph-mon[26394]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:39.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:38 smithi130 ceph-mon[21867]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:40.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:39 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T21:09:40.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:39 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:40.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:39 smithi130 ceph-mon[26394]: Deploying daemon osd.7 on smithi150 2024-04-22T21:09:40.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:39 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T21:09:40.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:39 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:40.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:39 smithi130 ceph-mon[21867]: Deploying daemon osd.7 on smithi150 2024-04-22T21:09:40.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:39 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T21:09:40.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:39 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:40.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:39 smithi150 ceph-mon[22676]: Deploying daemon osd.7 on smithi150 2024-04-22T21:09:41.220 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:40 smithi150 ceph-mon[22676]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:41.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:40 smithi130 ceph-mon[21867]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:41.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:40 smithi130 ceph-mon[26394]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:42.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:42 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:09:42.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:42 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:09:42.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:09:42.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:42 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:09:42.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:42 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:09:42.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:42 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:09:43.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:43 smithi130 ceph-mon[21867]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:43.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:43 smithi130 ceph-mon[26394]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:43.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:43 smithi150 ceph-mon[22676]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:45.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:45 smithi130 ceph-mon[21867]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:45.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:45 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:45.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:45 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:45.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:45 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:45.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:45 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:45.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:45 smithi130 ceph-mon[26394]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:45.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:45 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:45.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:45 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:45.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:45 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:45.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:45 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:45.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:45 smithi150 ceph-mon[22676]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:45.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:45 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:45.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:45 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:45.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:45 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:45.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:45 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:47.194 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:47 smithi150 ceph-mon[22676]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:47.228 INFO:teuthology.orchestra.run.smithi150.stdout:Created osd(s) 7 on host 'smithi150' 2024-04-22T21:09:47.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:47 smithi130 ceph-mon[21867]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:47.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:47 smithi130 ceph-mon[26394]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:47.870 DEBUG:teuthology.orchestra.run.smithi150:osd.7> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.7.service 2024-04-22T21:09:47.872 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-22T21:09:47.873 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-22T21:09:48.456 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:48 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:48.456 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:48 smithi130 ceph-mon[26394]: from='osd.7 [v2:172.21.15.150:6824/2168510914,v1:172.21.15.150:6825/2168510914]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T21:09:48.456 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:48 smithi130 ceph-mon[26394]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T21:09:48.457 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:48 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:48.457 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:48 smithi130 ceph-mon[21867]: from='osd.7 [v2:172.21.15.150:6824/2168510914,v1:172.21.15.150:6825/2168510914]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T21:09:48.457 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:48 smithi130 ceph-mon[21867]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T21:09:48.499 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:48 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:48.499 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:48 smithi150 ceph-mon[22676]: from='osd.7 [v2:172.21.15.150:6824/2168510914,v1:172.21.15.150:6825/2168510914]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T21:09:48.500 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:48 smithi150 ceph-mon[22676]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T21:09:49.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:49 smithi150 ceph-mon[22676]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:49.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:49 smithi150 ceph-mon[22676]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T21:09:49.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:49 smithi150 ceph-mon[22676]: osdmap e52: 8 total, 7 up, 8 in 2024-04-22T21:09:49.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:49 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:49.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:49 smithi150 ceph-mon[22676]: from='osd.7 [v2:172.21.15.150:6824/2168510914,v1:172.21.15.150:6825/2168510914]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:49.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:49 smithi150 ceph-mon[22676]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:49.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:49 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:49.363 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:09:49 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[34862]: 2024-04-22T21:09:49.252+0000 7fc5304a6700 -1 osd.7 0 waiting for initial osdmap 2024-04-22T21:09:49.363 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:09:49 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[34862]: 2024-04-22T21:09:49.259+0000 7fc52963a700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:09:49.473 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[26394]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:49.473 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[26394]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T21:09:49.473 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[26394]: osdmap e52: 8 total, 7 up, 8 in 2024-04-22T21:09:49.473 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:49.473 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[26394]: from='osd.7 [v2:172.21.15.150:6824/2168510914,v1:172.21.15.150:6825/2168510914]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:49.474 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[26394]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:49.474 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:49.474 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[21867]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:49.474 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[21867]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T21:09:49.474 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[21867]: osdmap e52: 8 total, 7 up, 8 in 2024-04-22T21:09:49.474 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:49.474 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[21867]: from='osd.7 [v2:172.21.15.150:6824/2168510914,v1:172.21.15.150:6825/2168510914]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:49.474 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[21867]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:09:49.474 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:49 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:49.874 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:09:50.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:50 smithi130 ceph-mon[26394]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-22T21:09:50.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:50 smithi130 ceph-mon[26394]: osdmap e53: 8 total, 7 up, 8 in 2024-04-22T21:09:50.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:50 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:50.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:50 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:50.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:50 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/938783133' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T21:09:50.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:50 smithi130 ceph-mon[21867]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-22T21:09:50.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:50 smithi130 ceph-mon[21867]: osdmap e53: 8 total, 7 up, 8 in 2024-04-22T21:09:50.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:50 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:50.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:50 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:50.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:50 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/938783133' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T21:09:50.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:50 smithi150 ceph-mon[22676]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-22T21:09:50.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:50 smithi150 ceph-mon[22676]: osdmap e53: 8 total, 7 up, 8 in 2024-04-22T21:09:50.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:50 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:50.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:50 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:50.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:50 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/938783133' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T21:09:50.754 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1713820165,"num_in_osds":8,"osd_in_since":1713820172,"num_remapped_pgs":0} 2024-04-22T21:09:51.529 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:51 smithi150 ceph-mon[22676]: purged_snaps scrub starts 2024-04-22T21:09:51.530 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:51 smithi150 ceph-mon[22676]: purged_snaps scrub ok 2024-04-22T21:09:51.530 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:51 smithi150 ceph-mon[22676]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:51.530 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:51 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:51.530 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:51 smithi150 ceph-mon[22676]: osd.7 [v2:172.21.15.150:6824/2168510914,v1:172.21.15.150:6825/2168510914] boot 2024-04-22T21:09:51.530 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:51 smithi150 ceph-mon[22676]: osdmap e54: 8 total, 8 up, 8 in 2024-04-22T21:09:51.530 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:51 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:51.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[21867]: purged_snaps scrub starts 2024-04-22T21:09:51.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[21867]: purged_snaps scrub ok 2024-04-22T21:09:51.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[21867]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:51.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:51.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[21867]: osd.7 [v2:172.21.15.150:6824/2168510914,v1:172.21.15.150:6825/2168510914] boot 2024-04-22T21:09:51.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[21867]: osdmap e54: 8 total, 8 up, 8 in 2024-04-22T21:09:51.559 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:51.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[26394]: purged_snaps scrub starts 2024-04-22T21:09:51.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[26394]: purged_snaps scrub ok 2024-04-22T21:09:51.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[26394]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T21:09:51.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:51.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[26394]: osd.7 [v2:172.21.15.150:6824/2168510914,v1:172.21.15.150:6825/2168510914] boot 2024-04-22T21:09:51.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[26394]: osdmap e54: 8 total, 8 up, 8 in 2024-04-22T21:09:51.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:51 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:09:51.755 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-22T21:09:52.389 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: osdmap e55: 8 total, 8 up, 8 in 2024-04-22T21:09:52.389 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: Detected new or changed devices on smithi150 2024-04-22T21:09:52.389 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:52.389 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:52.389 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:52.389 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:52.389 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:52.389 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: Adjusting osd_memory_target on smithi150 to 4272M 2024-04-22T21:09:52.390 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:52.390 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:52.390 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:52.390 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:52.390 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:52.390 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:52.390 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: osdmap e55: 8 total, 8 up, 8 in 2024-04-22T21:09:52.390 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: Detected new or changed devices on smithi150 2024-04-22T21:09:52.390 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:52.390 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:52.390 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:52.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:52.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:52.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: Adjusting osd_memory_target on smithi150 to 4272M 2024-04-22T21:09:52.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:52.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:52.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:52.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:52.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:52.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:52 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:52.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: osdmap e55: 8 total, 8 up, 8 in 2024-04-22T21:09:52.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: Detected new or changed devices on smithi150 2024-04-22T21:09:52.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:52.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:52.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:52.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:52.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:09:52.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: Adjusting osd_memory_target on smithi150 to 4272M 2024-04-22T21:09:52.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:52.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:52.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:09:52.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:09:52.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:09:52.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:52 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:09:53.400 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:53 smithi130 ceph-mon[26394]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:09:53.400 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:53 smithi130 ceph-mon[26394]: osdmap e56: 8 total, 8 up, 8 in 2024-04-22T21:09:53.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:53 smithi130 ceph-mon[21867]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:09:53.401 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:53 smithi130 ceph-mon[21867]: osdmap e56: 8 total, 8 up, 8 in 2024-04-22T21:09:53.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:53 smithi150 ceph-mon[22676]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:09:53.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:53 smithi150 ceph-mon[22676]: osdmap e56: 8 total, 8 up, 8 in 2024-04-22T21:09:53.765 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:09:54.384 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:54 smithi130 ceph-mon[21867]: osdmap e57: 8 total, 8 up, 8 in 2024-04-22T21:09:54.384 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:54 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/959056281' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T21:09:54.384 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:54 smithi130 ceph-mon[26394]: osdmap e57: 8 total, 8 up, 8 in 2024-04-22T21:09:54.384 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:54 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/959056281' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T21:09:54.407 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1713820190,"num_in_osds":8,"osd_in_since":1713820172,"num_remapped_pgs":0} 2024-04-22T21:09:54.407 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T21:09:54.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:54 smithi150 ceph-mon[22676]: osdmap e57: 8 total, 8 up, 8 in 2024-04-22T21:09:54.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:54 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/959056281' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T21:09:54.617 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:09:55.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:55 smithi130 ceph-mon[21867]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:09:55.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:55 smithi130 ceph-mon[21867]: osdmap e58: 8 total, 8 up, 8 in 2024-04-22T21:09:55.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:55 smithi130 ceph-mon[26394]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:09:55.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:55 smithi130 ceph-mon[26394]: osdmap e58: 8 total, 8 up, 8 in 2024-04-22T21:09:55.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:55 smithi150 ceph-mon[22676]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:09:55.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:55 smithi150 ceph-mon[22676]: osdmap e58: 8 total, 8 up, 8 in 2024-04-22T21:09:56.378 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:09:56.378 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":58,"fsid":"dd7d46b8-00eb-11ef-bc93-c7b262605968","created":"2024-04-22T21:04:53.945301+0000","modified":"2024-04-22T21:09:54.271622+0000","last_up_change":"2024-04-22T21:09:50.249497+0000","last_in_change":"2024-04-22T21:09:32.660425+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T21:07:52.066657+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b8e1579b-9111-4278-8801-1892b56c913a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1555952224},{"type":"v1","addr":"172.21.15.130:6803","nonce":1555952224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1555952224},{"type":"v1","addr":"172.21.15.130:6805","nonce":1555952224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":1555952224},{"type":"v1","addr":"172.21.15.130:6809","nonce":1555952224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1555952224},{"type":"v1","addr":"172.21.15.130:6807","nonce":1555952224}]},"public_addr":"172.21.15.130:6803/1555952224","cluster_addr":"172.21.15.130:6805/1555952224","heartbeat_back_addr":"172.21.15.130:6809/1555952224","heartbeat_front_addr":"172.21.15.130:6807/1555952224","state":["exists","up"]},{"osd":1,"uuid":"189889ca-c6b5-4c2c-a3f8-082e23fe4669","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":456112775},{"type":"v1","addr":"172.21.15.130:6811","nonce":456112775}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":456112775},{"type":"v1","addr":"172.21.15.130:6813","nonce":456112775}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":456112775},{"type":"v1","addr":"172.21.15.130:6817","nonce":456112775}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":456112775},{"type":"v1","addr":"172.21.15.130:6815","nonce":456112775}]},"public_addr":"172.21.15.130:6811/456112775","cluster_addr":"172.21.15.130:6813/456112775","heartbeat_back_addr":"172.21.15.130:6817/456112775","heartbeat_front_addr":"172.21.15.130:6815/456112775","state":["exists","up"]},{"osd":2,"uuid":"1993302f-f436-47e7-8cf5-7e00c0bdd1db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":802772272},{"type":"v1","addr":"172.21.15.130:6819","nonce":802772272}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":802772272},{"type":"v1","addr":"172.21.15.130:6821","nonce":802772272}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":802772272},{"type":"v1","addr":"172.21.15.130:6825","nonce":802772272}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":802772272},{"type":"v1","addr":"172.21.15.130:6823","nonce":802772272}]},"public_addr":"172.21.15.130:6819/802772272","cluster_addr":"172.21.15.130:6821/802772272","heartbeat_back_addr":"172.21.15.130:6825/802772272","heartbeat_front_addr":"172.21.15.130:6823/802772272","state":["exists","up"]},{"osd":3,"uuid":"9093e7a3-e43a-4387-bb67-c5f6eff53bae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":929040654},{"type":"v1","addr":"172.21.15.130:6827","nonce":929040654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":929040654},{"type":"v1","addr":"172.21.15.130:6829","nonce":929040654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":929040654},{"type":"v1","addr":"172.21.15.130:6833","nonce":929040654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":929040654},{"type":"v1","addr":"172.21.15.130:6831","nonce":929040654}]},"public_addr":"172.21.15.130:6827/929040654","cluster_addr":"172.21.15.130:6829/929040654","heartbeat_back_addr":"172.21.15.130:6833/929040654","heartbeat_front_addr":"172.21.15.130:6831/929040654","state":["exists","up"]},{"osd":4,"uuid":"872ec63d-c15e-4680-ab71-e1cf974154cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":3400040487},{"type":"v1","addr":"172.21.15.150:6801","nonce":3400040487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":3400040487},{"type":"v1","addr":"172.21.15.150:6803","nonce":3400040487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":3400040487},{"type":"v1","addr":"172.21.15.150:6807","nonce":3400040487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":3400040487},{"type":"v1","addr":"172.21.15.150:6805","nonce":3400040487}]},"public_addr":"172.21.15.150:6801/3400040487","cluster_addr":"172.21.15.150:6803/3400040487","heartbeat_back_addr":"172.21.15.150:6807/3400040487","heartbeat_front_addr":"172.21.15.150:6805/3400040487","state":["exists","up"]},{"osd":5,"uuid":"67066d12-b377-4cc6-93f0-ce22e7fb1de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":3183024588},{"type":"v1","addr":"172.21.15.150:6809","nonce":3183024588}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3183024588},{"type":"v1","addr":"172.21.15.150:6811","nonce":3183024588}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3183024588},{"type":"v1","addr":"172.21.15.150:6815","nonce":3183024588}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3183024588},{"type":"v1","addr":"172.21.15.150:6813","nonce":3183024588}]},"public_addr":"172.21.15.150:6809/3183024588","cluster_addr":"172.21.15.150:6811/3183024588","heartbeat_back_addr":"172.21.15.150:6815/3183024588","heartbeat_front_addr":"172.21.15.150:6813/3183024588","state":["exists","up"]},{"osd":6,"uuid":"8f18c163-a1a4-4016-9848-1d43db8a0c52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":3361358828},{"type":"v1","addr":"172.21.15.150:6817","nonce":3361358828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":3361358828},{"type":"v1","addr":"172.21.15.150:6819","nonce":3361358828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":3361358828},{"type":"v1","addr":"172.21.15.150:6823","nonce":3361358828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":3361358828},{"type":"v1","addr":"172.21.15.150:6821","nonce":3361358828}]},"public_addr":"172.21.15.150:6817/3361358828","cluster_addr":"172.21.15.150:6819/3361358828","heartbeat_back_addr":"172.21.15.150:6823/3361358828","heartbeat_front_addr":"172.21.15.150:6821/3361358828","state":["exists","up"]},{"osd":7,"uuid":"f45d779f-52bd-4994-b35d-61601dda32ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":2168510914},{"type":"v1","addr":"172.21.15.150:6825","nonce":2168510914}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":2168510914},{"type":"v1","addr":"172.21.15.150:6827","nonce":2168510914}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":2168510914},{"type":"v1","addr":"172.21.15.150:6831","nonce":2168510914}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":2168510914},{"type":"v1","addr":"172.21.15.150:6829","nonce":2168510914}]},"public_addr":"172.21.15.150:6825/2168510914","cluster_addr":"172.21.15.150:6827/2168510914","heartbeat_back_addr":"172.21.15.150:6831/2168510914","heartbeat_front_addr":"172.21.15.150:6829/2168510914","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:07:00.637423+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:07:24.770703+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:07:49.217681+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:08:13.854620+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:08:35.673654+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:08:59.692065+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:09:23.718847+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:09:48.470551+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/3442044605":"2024-04-23T21:05:41.702451+0000","172.21.15.130:6801/1754652716":"2024-04-23T21:05:41.702451+0000","172.21.15.130:6800/1814259082":"2024-04-23T21:05:19.464652+0000","172.21.15.130:0/2290497716":"2024-04-23T21:05:41.702451+0000","172.21.15.130:0/636209570":"2024-04-23T21:05:19.464652+0000","172.21.15.130:0/2397194687":"2024-04-23T21:05:41.702451+0000","172.21.15.130:6801/1814259082":"2024-04-23T21:05:19.464652+0000","172.21.15.130:0/2845297619":"2024-04-23T21:05:19.464652+0000","172.21.15.130:6800/1754652716":"2024-04-23T21:05:41.702451+0000","172.21.15.130:0/4177177338":"2024-04-23T21:05:19.464652+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T21:09:57.152 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-22T21:07:52.066657+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-22T21:09:57.153 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-22T21:09:57.368 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:09:57.491 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:57 smithi130 ceph-mon[21867]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:09:57.491 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:57 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3564006823' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T21:09:57.491 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:57 smithi130 ceph-mon[26394]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:09:57.491 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:57 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3564006823' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T21:09:57.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:57 smithi150 ceph-mon[22676]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:09:57.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:57 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3564006823' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T21:09:59.279 INFO:teuthology.orchestra.run.smithi130.stdout:pg_num: 1 2024-04-22T21:09:59.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:09:59 smithi130 ceph-mon[21867]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T21:09:59.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:09:59 smithi130 ceph-mon[26394]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T21:09:59.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:09:59 smithi150 ceph-mon[22676]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T21:10:00.054 INFO:tasks.cephadm:Adding prometheus.a on smithi150 2024-04-22T21:10:00.054 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch apply prometheus '1;smithi150=a' 2024-04-22T21:10:00.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:00 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1340902566' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-22T21:10:00.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:00 smithi150 ceph-mon[22676]: overall HEALTH_OK 2024-04-22T21:10:00.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:00 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1340902566' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-22T21:10:00.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:00 smithi130 ceph-mon[21867]: overall HEALTH_OK 2024-04-22T21:10:00.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:00 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1340902566' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-22T21:10:00.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:00 smithi130 ceph-mon[26394]: overall HEALTH_OK 2024-04-22T21:10:01.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:01 smithi130 ceph-mon[21867]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-22T21:10:01.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:01 smithi130 ceph-mon[26394]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-22T21:10:01.596 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:01 smithi150 ceph-mon[22676]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-22T21:10:02.029 INFO:teuthology.orchestra.run.smithi150.stdout:Scheduled prometheus update... 2024-04-22T21:10:02.656 DEBUG:teuthology.orchestra.run.smithi150:prometheus.a> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a.service 2024-04-22T21:10:02.659 INFO:tasks.cephadm:Adding node-exporter.a on smithi130 2024-04-22T21:10:02.659 INFO:tasks.cephadm:Adding node-exporter.b on smithi150 2024-04-22T21:10:02.659 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch apply node-exporter '2;smithi130=a;smithi150=b' 2024-04-22T21:10:03.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[21867]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-22T21:10:03.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[21867]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi150=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:10:03.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[21867]: Saving service prometheus spec with placement smithi150=a;count:1 2024-04-22T21:10:03.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:10:03.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:10:03.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:10:03.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:10:03.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:10:03.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-22T21:10:03.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[26394]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-22T21:10:03.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[26394]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi150=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:10:03.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[26394]: Saving service prometheus spec with placement smithi150=a;count:1 2024-04-22T21:10:03.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:10:03.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:10:03.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:10:03.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:10:03.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:10:03.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-22T21:10:03.310 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ignoring --setuser ceph since I am not root 2024-04-22T21:10:03.310 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ignoring --setgroup ceph since I am not root 2024-04-22T21:10:03.310 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-mgr[22097]: -- 172.21.15.130:0/294978545 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555de51c2340 con 0x555de519ac00 2024-04-22T21:10:03.310 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:03.245+0000 7f4a7070d000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:10:03.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-mon[22676]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-22T21:10:03.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-mon[22676]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi150=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:10:03.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-mon[22676]: Saving service prometheus spec with placement smithi150=a;count:1 2024-04-22T21:10:03.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:10:03.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:10:03.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:10:03.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:10:03.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' 2024-04-22T21:10:03.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-22T21:10:03.321 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ignoring --setuser ceph since I am not root 2024-04-22T21:10:03.321 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ignoring --setgroup ceph since I am not root 2024-04-22T21:10:03.322 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-mgr[23856]: -- 172.21.15.150:0/2500918335 <== mon.2 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562cd3708340 con 0x562cd36e0c00 2024-04-22T21:10:03.322 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:03.239+0000 7f5148f51000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:10:03.611 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:03.378+0000 7f5148f51000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:10:03.611 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:03.476+0000 7f5148f51000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:10:03.766 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:03.372+0000 7f4a7070d000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:10:03.766 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:03.468+0000 7f4a7070d000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:10:03.766 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:03.771+0000 7f4a7070d000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:10:03.982 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:03.787+0000 7f5148f51000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:10:03.982 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:03.870+0000 7f5148f51000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:10:04.057 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:03.853+0000 7f4a7070d000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:10:04.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:04 smithi150 ceph-mon[22676]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-22T21:10:04.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:04 smithi150 ceph-mon[22676]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-22T21:10:04.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:04 smithi130 ceph-mon[21867]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-22T21:10:04.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:04 smithi130 ceph-mon[21867]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-22T21:10:04.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:04 smithi130 ceph-mon[26394]: from='mgr.14152 172.21.15.130:0/3611758066' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-22T21:10:04.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:04 smithi130 ceph-mon[26394]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-22T21:10:05.307 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:04 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:04.966+0000 7f4a7070d000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:10:05.307 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:05.131+0000 7f4a7070d000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:10:05.308 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:05.212+0000 7f4a7070d000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:10:05.361 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:05.016+0000 7f5148f51000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:10:05.361 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:05.181+0000 7f5148f51000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:10:05.361 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:05.262+0000 7f5148f51000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:10:05.807 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:05.523+0000 7f4a7070d000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:10:05.808 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:05.613+0000 7f4a7070d000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:10:05.808 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:05.706+0000 7f4a7070d000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:10:05.861 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:05.575+0000 7f5148f51000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:10:05.861 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:05.665+0000 7f5148f51000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:10:05.861 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:05.758+0000 7f5148f51000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:10:06.307 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:05.947+0000 7f4a7070d000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:10:06.361 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:06.000+0000 7f5148f51000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:10:06.807 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:06.384+0000 7f4a7070d000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:10:06.861 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:06.436+0000 7f5148f51000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:10:07.250 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:06.871+0000 7f4a7070d000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:10:07.250 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:06.964+0000 7f4a7070d000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:10:07.298 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:06.922+0000 7f5148f51000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:10:07.299 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:07 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:07.015+0000 7f5148f51000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:10:07.299 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:07 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:07.304+0000 7f5148f51000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:10:07.557 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:07.255+0000 7f4a7070d000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:10:07.557 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:07.345+0000 7f4a7070d000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:10:07.611 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:07 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:07.393+0000 7f5148f51000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:10:08.057 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:07.748+0000 7f4a7070d000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:10:08.057 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:07.879+0000 7f4a7070d000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:10:08.057 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:07.971+0000 7f4a7070d000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:10:08.111 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:07 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:07.797+0000 7f5148f51000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:10:08.111 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:07 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:07.929+0000 7f5148f51000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:10:08.111 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:08.022+0000 7f5148f51000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:10:08.557 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:08.085+0000 7f4a7070d000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:10:08.557 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:08.175+0000 7f4a7070d000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:10:08.614 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:08.136+0000 7f5148f51000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:10:08.614 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:08.225+0000 7f5148f51000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:10:09.057 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:08.721+0000 7f4a7070d000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:10:09.111 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:08.768+0000 7f5148f51000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:10:09.557 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:09.084+0000 7f4a7070d000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:10:09.611 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:09.131+0000 7f5148f51000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:10:09.611 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: [22/Apr/2024:21:10:09] ENGINE Bus STARTING 2024-04-22T21:10:09.611 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: CherryPy Checker: 2024-04-22T21:10:09.611 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: The Application mounted at '' has an empty config. 2024-04-22T21:10:09.612 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: 2024-04-22T21:10:09.612 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: [22/Apr/2024:21:10:09] ENGINE Serving on http://:::9283 2024-04-22T21:10:09.612 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: [22/Apr/2024:21:10:09] ENGINE Bus STARTED 2024-04-22T21:10:10.253 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:10 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:10:10] ENGINE Bus STARTING 2024-04-22T21:10:10.254 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:10 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: CherryPy Checker: 2024-04-22T21:10:10.254 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:10 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: The Application mounted at '' has an empty config. 2024-04-22T21:10:10.254 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:10 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:10:10.254 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:10 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:10:10] ENGINE Serving on http://:::9283 2024-04-22T21:10:10.254 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:10 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:10:10] ENGINE Bus STARTED 2024-04-22T21:10:10.254 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[26394]: Active manager daemon y restarted 2024-04-22T21:10:10.254 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[26394]: Activating manager daemon y 2024-04-22T21:10:10.254 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[26394]: osdmap e59: 8 total, 8 up, 8 in 2024-04-22T21:10:10.255 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[26394]: Standby manager daemon x restarted 2024-04-22T21:10:10.255 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[26394]: Standby manager daemon x started 2024-04-22T21:10:10.255 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/4272381141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:10:10.255 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/4272381141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:10:10.255 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/4272381141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:10:10.255 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/4272381141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:10:10.255 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[21867]: Active manager daemon y restarted 2024-04-22T21:10:10.255 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[21867]: Activating manager daemon y 2024-04-22T21:10:10.255 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[21867]: osdmap e59: 8 total, 8 up, 8 in 2024-04-22T21:10:10.255 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[21867]: Standby manager daemon x restarted 2024-04-22T21:10:10.255 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[21867]: Standby manager daemon x started 2024-04-22T21:10:10.255 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/4272381141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:10:10.256 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/4272381141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:10:10.256 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/4272381141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:10:10.256 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:09 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/4272381141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:10:10.288 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-mon[22676]: Active manager daemon y restarted 2024-04-22T21:10:10.288 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-mon[22676]: Activating manager daemon y 2024-04-22T21:10:10.288 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-mon[22676]: osdmap e59: 8 total, 8 up, 8 in 2024-04-22T21:10:10.288 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-mon[22676]: Standby manager daemon x restarted 2024-04-22T21:10:10.288 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-mon[22676]: Standby manager daemon x started 2024-04-22T21:10:10.288 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/4272381141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:10:10.288 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/4272381141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:10:10.289 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/4272381141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:10:10.289 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:09 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/4272381141' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:10:10.557 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:10 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:10:10] ENGINE Bus STARTING 2024-04-22T21:10:10.557 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:10 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:10:10] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:10:10.557 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:10 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:10:10] ENGINE Bus STARTED 2024-04-22T21:10:11.008 INFO:teuthology.orchestra.run.smithi150.stdout:Scheduled node-exporter update... 2024-04-22T21:10:11.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: mgrmap e17: y(active, starting, since 0.908972s), standbys: x 2024-04-22T21:10:11.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:10:11.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:10:11.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:10:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:10:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:10:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:10:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:10:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:10:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:10:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:10:11.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:10:11.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:10:11.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:10:11.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:10:11.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:10:11.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:10:11.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: Manager daemon y is now available 2024-04-22T21:10:11.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:10:11.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:10:11.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:10:11.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:10:11.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:11.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:10:11.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:10:11.063 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:11.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: mgrmap e17: y(active, starting, since 0.908972s), standbys: x 2024-04-22T21:10:11.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:10:11.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:10:11.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:10:11.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:10:11.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:10:11.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:10:11.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:10:11.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:10:11.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:10:11.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:10:11.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:10:11.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:10:11.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:10:11.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:10:11.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:10:11.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:10:11.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: Manager daemon y is now available 2024-04-22T21:10:11.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:10:11.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:10:11.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:10:11.065 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:10:11.065 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:11.065 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:10:11.065 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:10:11.065 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:11.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: mgrmap e17: y(active, starting, since 0.908972s), standbys: x 2024-04-22T21:10:11.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:10:11.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:10:11.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:10:11.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: Manager daemon y is now available 2024-04-22T21:10:11.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:10:11.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:10:11.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:10:11.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:10:11.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:11.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:10:11.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:10:11.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:11.846 DEBUG:teuthology.orchestra.run.smithi130:node-exporter.a> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.a.service 2024-04-22T21:10:11.848 DEBUG:teuthology.orchestra.run.smithi150:node-exporter.b> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.b.service 2024-04-22T21:10:11.850 INFO:tasks.cephadm:Adding alertmanager.a on smithi130 2024-04-22T21:10:11.851 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch apply alertmanager '1;smithi130=a' 2024-04-22T21:10:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:12 smithi150 ceph-mon[22676]: [22/Apr/2024:21:10:10] ENGINE Bus STARTING 2024-04-22T21:10:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:12 smithi150 ceph-mon[22676]: [22/Apr/2024:21:10:10] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:10:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:12 smithi150 ceph-mon[22676]: [22/Apr/2024:21:10:10] ENGINE Bus STARTED 2024-04-22T21:10:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:12 smithi150 ceph-mon[22676]: mgrmap e18: y(active, since 1.91186s), standbys: x 2024-04-22T21:10:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:12 smithi150 ceph-mon[22676]: from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi130=a;smithi150=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:10:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:12 smithi150 ceph-mon[22676]: Saving service node-exporter spec with placement smithi130=a;smithi150=b;count:2 2024-04-22T21:10:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:12 smithi150 ceph-mon[22676]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:12 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:12 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:12 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:12.157 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[26394]: [22/Apr/2024:21:10:10] ENGINE Bus STARTING 2024-04-22T21:10:12.157 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[26394]: [22/Apr/2024:21:10:10] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:10:12.157 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[26394]: [22/Apr/2024:21:10:10] ENGINE Bus STARTED 2024-04-22T21:10:12.157 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[26394]: mgrmap e18: y(active, since 1.91186s), standbys: x 2024-04-22T21:10:12.157 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[26394]: from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi130=a;smithi150=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:10:12.157 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[26394]: Saving service node-exporter spec with placement smithi130=a;smithi150=b;count:2 2024-04-22T21:10:12.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[26394]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:12.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:12.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:12.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:12.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[21867]: [22/Apr/2024:21:10:10] ENGINE Bus STARTING 2024-04-22T21:10:12.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[21867]: [22/Apr/2024:21:10:10] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:10:12.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[21867]: [22/Apr/2024:21:10:10] ENGINE Bus STARTED 2024-04-22T21:10:12.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[21867]: mgrmap e18: y(active, since 1.91186s), standbys: x 2024-04-22T21:10:12.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[21867]: from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi130=a;smithi150=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:10:12.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[21867]: Saving service node-exporter spec with placement smithi130=a;smithi150=b;count:2 2024-04-22T21:10:12.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[21867]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:12.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:12.159 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:12.159 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:12 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:13.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:13 smithi150 ceph-mon[22676]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:10:13.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:13 smithi150 ceph-mon[22676]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:13.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:13 smithi150 ceph-mon[22676]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-22T21:10:13.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:13 smithi150 ceph-mon[22676]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:10:13.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:13 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:13.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:13 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:13.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:13 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:13.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:10:13.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[26394]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:13.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[26394]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-22T21:10:13.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:10:13.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:13.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:13.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:13.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[21867]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:10:13.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[21867]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:13.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[21867]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-22T21:10:13.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[21867]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:10:13.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:13.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:13.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:13 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:14.056 INFO:teuthology.orchestra.run.smithi150.stdout:Scheduled alertmanager update... 2024-04-22T21:10:14.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:14 smithi130 ceph-mon[21867]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:10:14.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:14 smithi130 ceph-mon[21867]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:10:14.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:14 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:14.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:14 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:14.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:14 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:10:14.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:14 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:10:14.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:14 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:14.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:14 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:14.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:14 smithi150 ceph-mon[22676]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:10:14.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:14 smithi150 ceph-mon[22676]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:10:14.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:14 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:14.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:14 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:14.776 DEBUG:teuthology.orchestra.run.smithi130:alertmanager.a> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@alertmanager.a.service 2024-04-22T21:10:14.779 INFO:tasks.cephadm:Adding grafana.a on smithi150 2024-04-22T21:10:14.779 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph orch apply grafana '1;smithi150=a' 2024-04-22T21:10:15.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:15 smithi150 ceph-mon[22676]: Deploying daemon prometheus.a on smithi150 2024-04-22T21:10:15.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:15 smithi150 ceph-mon[22676]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:15.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:15 smithi150 ceph-mon[22676]: from='client.34373 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi130=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:10:15.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:15 smithi150 ceph-mon[22676]: Saving service alertmanager spec with placement smithi130=a;count:1 2024-04-22T21:10:15.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:15 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:15.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:15 smithi130 ceph-mon[21867]: Deploying daemon prometheus.a on smithi150 2024-04-22T21:10:15.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:15 smithi130 ceph-mon[21867]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:15.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:15 smithi130 ceph-mon[21867]: from='client.34373 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi130=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:10:15.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:15 smithi130 ceph-mon[21867]: Saving service alertmanager spec with placement smithi130=a;count:1 2024-04-22T21:10:15.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:15 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:15.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:15 smithi130 ceph-mon[26394]: Deploying daemon prometheus.a on smithi150 2024-04-22T21:10:15.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:15 smithi130 ceph-mon[26394]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:15.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:15 smithi130 ceph-mon[26394]: from='client.34373 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi130=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:10:15.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:15 smithi130 ceph-mon[26394]: Saving service alertmanager spec with placement smithi130=a;count:1 2024-04-22T21:10:15.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:15 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:17.290 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:17 smithi150 ceph-mon[22676]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:17.360 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:17 smithi130 ceph-mon[26394]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:17.361 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:17 smithi130 ceph-mon[21867]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:17.466 INFO:teuthology.orchestra.run.smithi150.stdout:Scheduled grafana update... 2024-04-22T21:10:18.446 DEBUG:teuthology.orchestra.run.smithi150:grafana.a> sudo journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@grafana.a.service 2024-04-22T21:10:18.448 INFO:tasks.cephadm:Setting up client nodes... 2024-04-22T21:10:18.448 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-22T21:10:18.682 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:18 smithi150 ceph-mon[22676]: from='client.34379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi150=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:10:18.682 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:18 smithi150 ceph-mon[22676]: Saving service grafana spec with placement smithi150=a;count:1 2024-04-22T21:10:18.682 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:18 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:18.682 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:18 smithi150 ceph-mon[22676]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:18.758 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:18 smithi130 ceph-mon[21867]: from='client.34379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi150=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:10:18.758 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:18 smithi130 ceph-mon[21867]: Saving service grafana spec with placement smithi150=a;count:1 2024-04-22T21:10:18.759 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:18 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:18.759 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:18 smithi130 ceph-mon[21867]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:18.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:18 smithi130 ceph-mon[26394]: from='client.34379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi150=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:10:18.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:18 smithi130 ceph-mon[26394]: Saving service grafana spec with placement smithi150=a;count:1 2024-04-22T21:10:18.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:18 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:18.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:18 smithi130 ceph-mon[26394]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:20.628 INFO:teuthology.orchestra.run.smithi130.stdout:[client.0] 2024-04-22T21:10:20.628 INFO:teuthology.orchestra.run.smithi130.stdout: key = AQA80iZmQSWeJRAA20o3r/hm6DQ+3bJagRu8BQ== 2024-04-22T21:10:21.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:21 smithi130 ceph-mon[26394]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:21.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:21 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/428288254' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T21:10:21.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:21 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T21:10:21.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:21 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T21:10:21.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:21 smithi130 ceph-mon[21867]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:21.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:21 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/428288254' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T21:10:21.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:21 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T21:10:21.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:21 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T21:10:21.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:21 smithi150 ceph-mon[22676]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:21.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:21 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/428288254' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T21:10:21.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:21 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T21:10:21.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:21 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T21:10:21.444 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-04-22T21:10:21.444 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-22T21:10:21.445 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-22T21:10:21.483 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-22T21:10:23.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:23 smithi150 ceph-mon[22676]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:23.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:23 smithi130 ceph-mon[21867]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:23.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:23 smithi130 ceph-mon[26394]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:25.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:24 smithi130 ceph-mon[21867]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:25.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:24 smithi130 ceph-mon[26394]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:25.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:24 smithi150 ceph-mon[22676]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:26.533 INFO:teuthology.orchestra.run.smithi150.stdout:[client.1] 2024-04-22T21:10:26.533 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBC0iZmzdTIHxAAVLge8bNxot+fqTN8Qn7n8g== 2024-04-22T21:10:26.533 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:26 smithi150 systemd[1]: Starting Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:10:26.861 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:26 smithi150 podman[37746]: 2024-04-22 21:10:26.559856994 +0000 UTC m=+0.020468713 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T21:10:27.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:27 smithi130 ceph-mon[21867]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:27.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:27 smithi130 ceph-mon[21867]: from='client.? 172.21.15.150:0/3210025110' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T21:10:27.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:27 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T21:10:27.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:27 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T21:10:27.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:27 smithi130 ceph-mon[26394]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:27.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:27 smithi130 ceph-mon[26394]: from='client.? 172.21.15.150:0/3210025110' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T21:10:27.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:27 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T21:10:27.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:27 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T21:10:27.361 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:26 smithi150 podman[37746]: 2024-04-22 21:10:26.915429224 +0000 UTC m=+0.376040926 container create c106f1a0ac46d19a3392b0a5fa01459f9c4c5b4f6497f682cd978a27aa413285 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:10:27.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-mon[22676]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:27.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-mon[22676]: from='client.? 172.21.15.150:0/3210025110' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T21:10:27.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T21:10:27.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T21:10:27.420 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-22T21:10:27.420 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-22T21:10:27.420 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-22T21:10:27.457 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-22T21:10:27.458 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-22T21:10:27.458 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-22T21:10:27.664 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 podman[37746]: 2024-04-22 21:10:27.579089169 +0000 UTC m=+1.039700871 container init c106f1a0ac46d19a3392b0a5fa01459f9c4c5b4f6497f682cd978a27aa413285 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:10:27.664 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 podman[37746]: 2024-04-22 21:10:27.58224425 +0000 UTC m=+1.042855962 container start c106f1a0ac46d19a3392b0a5fa01459f9c4c5b4f6497f682cd978a27aa413285 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:10:27.664 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.606Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T21:10:27.664 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.606Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T21:10:27.664 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.606Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T21:10:27.664 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.606Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi150 (none))" 2024-04-22T21:10:27.664 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.606Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T21:10:27.665 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.606Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T21:10:27.665 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.608Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T21:10:27.665 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.608Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T21:10:27.665 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.609Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T21:10:27.665 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.611Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T21:10:27.665 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.611Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.826µs 2024-04-22T21:10:27.665 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.611Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T21:10:27.665 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.612Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-22T21:10:27.665 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.612Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.07µs wal_replay_duration=133.616µs total_replay_duration=169.508µs 2024-04-22T21:10:27.665 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.612Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T21:10:27.665 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.612Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T21:10:27.665 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.612Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T21:10:27.665 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.627Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.828645ms db_storage=590ns remote_storage=1.18µs web_handler=353ns query_engine=469ns scrape=407.475µs scrape_sd=16.294µs notify=1.018µs notify_sd=1.898µs rules=14.234706ms 2024-04-22T21:10:27.666 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:10:27.627Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T21:10:27.666 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 bash[37746]: c106f1a0ac46d19a3392b0a5fa01459f9c4c5b4f6497f682cd978a27aa413285 2024-04-22T21:10:27.679 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:28.111 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:10:27 smithi150 systemd[1]: Started Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:10:29.017 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:28 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:29.017 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:28 smithi130 ceph-mon[26394]: Deploying daemon node-exporter.a on smithi130 2024-04-22T21:10:29.017 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:28 smithi130 ceph-mon[26394]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:29.017 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:28 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:29.017 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:28 smithi130 ceph-mon[21867]: Deploying daemon node-exporter.a on smithi130 2024-04-22T21:10:29.017 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:28 smithi130 ceph-mon[21867]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:29.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:28 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:29.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:28 smithi150 ceph-mon[22676]: Deploying daemon node-exporter.a on smithi130 2024-04-22T21:10:29.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:28 smithi150 ceph-mon[22676]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:29.783 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:10:30.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:29 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1724518807' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T21:10:30.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:29 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1724518807' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T21:10:30.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:29 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1724518807' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T21:10:30.539 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":19,"active_gid":14445,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":2780444297},{"type":"v1","addr":"172.21.15.130:6801","nonce":2780444297}]},"active_addr":"172.21.15.130:6801/2780444297","active_change":"2024-04-22T21:10:09.087057+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":34334,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":59,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":840813444}]},{"addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":995961703}]},{"addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":3105135989}]},{"addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":2002813143}]}]}} 2024-04-22T21:10:30.546 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-22T21:10:30.547 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-22T21:10:30.547 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T21:10:30.760 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:30.807 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:30 smithi130 bash[41343]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-22T21:10:31.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:31 smithi130 ceph-mon[26394]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:31.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:31 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:31.204 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:31 smithi130 ceph-mon[21867]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:31.204 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:31 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:31.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:31 smithi150 ceph-mon[22676]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:31.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:31 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:32.998 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:32 smithi130 bash[41343]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-22T21:10:33.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:33 smithi130 ceph-mon[26394]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:33.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:33 smithi130 ceph-mon[21867]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:33.307 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:33 smithi130 bash[41343]: Writing manifest to image destination 2024-04-22T21:10:33.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:33 smithi150 ceph-mon[22676]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:33.448 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:10:33.448 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":59,"fsid":"dd7d46b8-00eb-11ef-bc93-c7b262605968","created":"2024-04-22T21:04:53.945301+0000","modified":"2024-04-22T21:10:09.086687+0000","last_up_change":"2024-04-22T21:09:50.249497+0000","last_in_change":"2024-04-22T21:09:32.660425+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T21:07:52.066657+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b8e1579b-9111-4278-8801-1892b56c913a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1555952224},{"type":"v1","addr":"172.21.15.130:6803","nonce":1555952224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1555952224},{"type":"v1","addr":"172.21.15.130:6805","nonce":1555952224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":1555952224},{"type":"v1","addr":"172.21.15.130:6809","nonce":1555952224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1555952224},{"type":"v1","addr":"172.21.15.130:6807","nonce":1555952224}]},"public_addr":"172.21.15.130:6803/1555952224","cluster_addr":"172.21.15.130:6805/1555952224","heartbeat_back_addr":"172.21.15.130:6809/1555952224","heartbeat_front_addr":"172.21.15.130:6807/1555952224","state":["exists","up"]},{"osd":1,"uuid":"189889ca-c6b5-4c2c-a3f8-082e23fe4669","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":456112775},{"type":"v1","addr":"172.21.15.130:6811","nonce":456112775}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":456112775},{"type":"v1","addr":"172.21.15.130:6813","nonce":456112775}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":456112775},{"type":"v1","addr":"172.21.15.130:6817","nonce":456112775}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":456112775},{"type":"v1","addr":"172.21.15.130:6815","nonce":456112775}]},"public_addr":"172.21.15.130:6811/456112775","cluster_addr":"172.21.15.130:6813/456112775","heartbeat_back_addr":"172.21.15.130:6817/456112775","heartbeat_front_addr":"172.21.15.130:6815/456112775","state":["exists","up"]},{"osd":2,"uuid":"1993302f-f436-47e7-8cf5-7e00c0bdd1db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":802772272},{"type":"v1","addr":"172.21.15.130:6819","nonce":802772272}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":802772272},{"type":"v1","addr":"172.21.15.130:6821","nonce":802772272}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":802772272},{"type":"v1","addr":"172.21.15.130:6825","nonce":802772272}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":802772272},{"type":"v1","addr":"172.21.15.130:6823","nonce":802772272}]},"public_addr":"172.21.15.130:6819/802772272","cluster_addr":"172.21.15.130:6821/802772272","heartbeat_back_addr":"172.21.15.130:6825/802772272","heartbeat_front_addr":"172.21.15.130:6823/802772272","state":["exists","up"]},{"osd":3,"uuid":"9093e7a3-e43a-4387-bb67-c5f6eff53bae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":929040654},{"type":"v1","addr":"172.21.15.130:6827","nonce":929040654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":929040654},{"type":"v1","addr":"172.21.15.130:6829","nonce":929040654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":929040654},{"type":"v1","addr":"172.21.15.130:6833","nonce":929040654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":929040654},{"type":"v1","addr":"172.21.15.130:6831","nonce":929040654}]},"public_addr":"172.21.15.130:6827/929040654","cluster_addr":"172.21.15.130:6829/929040654","heartbeat_back_addr":"172.21.15.130:6833/929040654","heartbeat_front_addr":"172.21.15.130:6831/929040654","state":["exists","up"]},{"osd":4,"uuid":"872ec63d-c15e-4680-ab71-e1cf974154cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":3400040487},{"type":"v1","addr":"172.21.15.150:6801","nonce":3400040487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":3400040487},{"type":"v1","addr":"172.21.15.150:6803","nonce":3400040487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":3400040487},{"type":"v1","addr":"172.21.15.150:6807","nonce":3400040487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":3400040487},{"type":"v1","addr":"172.21.15.150:6805","nonce":3400040487}]},"public_addr":"172.21.15.150:6801/3400040487","cluster_addr":"172.21.15.150:6803/3400040487","heartbeat_back_addr":"172.21.15.150:6807/3400040487","heartbeat_front_addr":"172.21.15.150:6805/3400040487","state":["exists","up"]},{"osd":5,"uuid":"67066d12-b377-4cc6-93f0-ce22e7fb1de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":3183024588},{"type":"v1","addr":"172.21.15.150:6809","nonce":3183024588}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3183024588},{"type":"v1","addr":"172.21.15.150:6811","nonce":3183024588}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3183024588},{"type":"v1","addr":"172.21.15.150:6815","nonce":3183024588}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3183024588},{"type":"v1","addr":"172.21.15.150:6813","nonce":3183024588}]},"public_addr":"172.21.15.150:6809/3183024588","cluster_addr":"172.21.15.150:6811/3183024588","heartbeat_back_addr":"172.21.15.150:6815/3183024588","heartbeat_front_addr":"172.21.15.150:6813/3183024588","state":["exists","up"]},{"osd":6,"uuid":"8f18c163-a1a4-4016-9848-1d43db8a0c52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":3361358828},{"type":"v1","addr":"172.21.15.150:6817","nonce":3361358828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":3361358828},{"type":"v1","addr":"172.21.15.150:6819","nonce":3361358828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":3361358828},{"type":"v1","addr":"172.21.15.150:6823","nonce":3361358828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":3361358828},{"type":"v1","addr":"172.21.15.150:6821","nonce":3361358828}]},"public_addr":"172.21.15.150:6817/3361358828","cluster_addr":"172.21.15.150:6819/3361358828","heartbeat_back_addr":"172.21.15.150:6823/3361358828","heartbeat_front_addr":"172.21.15.150:6821/3361358828","state":["exists","up"]},{"osd":7,"uuid":"f45d779f-52bd-4994-b35d-61601dda32ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":2168510914},{"type":"v1","addr":"172.21.15.150:6825","nonce":2168510914}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":2168510914},{"type":"v1","addr":"172.21.15.150:6827","nonce":2168510914}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":2168510914},{"type":"v1","addr":"172.21.15.150:6831","nonce":2168510914}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":2168510914},{"type":"v1","addr":"172.21.15.150:6829","nonce":2168510914}]},"public_addr":"172.21.15.150:6825/2168510914","cluster_addr":"172.21.15.150:6827/2168510914","heartbeat_back_addr":"172.21.15.150:6831/2168510914","heartbeat_front_addr":"172.21.15.150:6829/2168510914","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:07:00.637423+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:07:24.770703+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:07:49.217681+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:08:13.854620+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:08:35.673654+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:08:59.692065+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:09:23.718847+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:09:48.470551+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/4195343556":"2024-04-23T21:10:09.086654+0000","172.21.15.130:0/1551796354":"2024-04-23T21:10:09.086654+0000","172.21.15.130:0/3442044605":"2024-04-23T21:05:41.702451+0000","172.21.15.130:6800/4050722828":"2024-04-23T21:10:09.086654+0000","172.21.15.130:6801/1754652716":"2024-04-23T21:05:41.702451+0000","172.21.15.130:6800/1814259082":"2024-04-23T21:05:19.464652+0000","172.21.15.130:6801/4050722828":"2024-04-23T21:10:09.086654+0000","172.21.15.130:0/2290497716":"2024-04-23T21:05:41.702451+0000","172.21.15.130:0/636209570":"2024-04-23T21:05:19.464652+0000","172.21.15.130:0/2397194687":"2024-04-23T21:05:41.702451+0000","172.21.15.130:6801/1814259082":"2024-04-23T21:05:19.464652+0000","172.21.15.130:0/2845297619":"2024-04-23T21:05:19.464652+0000","172.21.15.130:0/4252030715":"2024-04-23T21:10:09.086654+0000","172.21.15.130:0/3720321995":"2024-04-23T21:10:09.086654+0000","172.21.15.130:6800/1754652716":"2024-04-23T21:05:41.702451+0000","172.21.15.130:0/4177177338":"2024-04-23T21:05:19.464652+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T21:10:33.807 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:33 smithi130 podman[41343]: 2024-04-22 21:10:33.429972333 +0000 UTC m=+3.863539150 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-22T21:10:33.807 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:33 smithi130 podman[41343]: 2024-04-22 21:10:33.545804479 +0000 UTC m=+3.979371338 container create 82b1ae40160ec66f6730ad62a2dc36e162e31717a94ab12e6024e2f8dcaf5cb8 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:10:34.092 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3147219396' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T21:10:34.092 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3147219396' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T21:10:34.093 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:33 smithi130 podman[41343]: 2024-04-22 21:10:33.998956535 +0000 UTC m=+4.432523349 container init 82b1ae40160ec66f6730ad62a2dc36e162e31717a94ab12e6024e2f8dcaf5cb8 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:10:34.093 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 podman[41343]: 2024-04-22 21:10:34.002207808 +0000 UTC m=+4.435774630 container start 82b1ae40160ec66f6730ad62a2dc36e162e31717a94ab12e6024e2f8dcaf5cb8 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:10:34.093 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.006Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-22T21:10:34.093 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.006Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-22T21:10:34.093 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.007Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-22T21:10:34.093 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T21:10:34.093 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-22T21:10:34.093 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=arp 2024-04-22T21:10:34.093 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-22T21:10:34.093 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-22T21:10:34.093 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=edac 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-22T21:10:34.094 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=os 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-22T21:10:34.095 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=stat 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=time 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=uname 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.008Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[41551]: ts=2024-04-22T21:10:34.009Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-22T21:10:34.096 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 bash[41343]: 82b1ae40160ec66f6730ad62a2dc36e162e31717a94ab12e6024e2f8dcaf5cb8 2024-04-22T21:10:34.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:34 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3147219396' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T21:10:34.557 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:10:34 smithi130 systemd[1]: Started Ceph node-exporter.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:10:34.589 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-22T21:10:34.589 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T21:10:34.803 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:35.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:35 smithi150 ceph-mon[22676]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:35.111 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:35 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:35.249 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:35 smithi130 ceph-mon[21867]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:35.249 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:35 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:35.249 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:35 smithi130 ceph-mon[26394]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:35.249 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:35 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:35.861 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:35 smithi150 bash[38020]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-22T21:10:36.285 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:36 smithi130 ceph-mon[21867]: Deploying daemon node-exporter.b on smithi150 2024-04-22T21:10:36.285 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:36 smithi130 ceph-mon[26394]: Deploying daemon node-exporter.b on smithi150 2024-04-22T21:10:36.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:36 smithi150 ceph-mon[22676]: Deploying daemon node-exporter.b on smithi150 2024-04-22T21:10:36.668 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:10:36.668 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":59,"fsid":"dd7d46b8-00eb-11ef-bc93-c7b262605968","created":"2024-04-22T21:04:53.945301+0000","modified":"2024-04-22T21:10:09.086687+0000","last_up_change":"2024-04-22T21:09:50.249497+0000","last_in_change":"2024-04-22T21:09:32.660425+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T21:07:52.066657+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b8e1579b-9111-4278-8801-1892b56c913a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1555952224},{"type":"v1","addr":"172.21.15.130:6803","nonce":1555952224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1555952224},{"type":"v1","addr":"172.21.15.130:6805","nonce":1555952224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":1555952224},{"type":"v1","addr":"172.21.15.130:6809","nonce":1555952224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1555952224},{"type":"v1","addr":"172.21.15.130:6807","nonce":1555952224}]},"public_addr":"172.21.15.130:6803/1555952224","cluster_addr":"172.21.15.130:6805/1555952224","heartbeat_back_addr":"172.21.15.130:6809/1555952224","heartbeat_front_addr":"172.21.15.130:6807/1555952224","state":["exists","up"]},{"osd":1,"uuid":"189889ca-c6b5-4c2c-a3f8-082e23fe4669","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":456112775},{"type":"v1","addr":"172.21.15.130:6811","nonce":456112775}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":456112775},{"type":"v1","addr":"172.21.15.130:6813","nonce":456112775}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":456112775},{"type":"v1","addr":"172.21.15.130:6817","nonce":456112775}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":456112775},{"type":"v1","addr":"172.21.15.130:6815","nonce":456112775}]},"public_addr":"172.21.15.130:6811/456112775","cluster_addr":"172.21.15.130:6813/456112775","heartbeat_back_addr":"172.21.15.130:6817/456112775","heartbeat_front_addr":"172.21.15.130:6815/456112775","state":["exists","up"]},{"osd":2,"uuid":"1993302f-f436-47e7-8cf5-7e00c0bdd1db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":802772272},{"type":"v1","addr":"172.21.15.130:6819","nonce":802772272}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":802772272},{"type":"v1","addr":"172.21.15.130:6821","nonce":802772272}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":802772272},{"type":"v1","addr":"172.21.15.130:6825","nonce":802772272}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":802772272},{"type":"v1","addr":"172.21.15.130:6823","nonce":802772272}]},"public_addr":"172.21.15.130:6819/802772272","cluster_addr":"172.21.15.130:6821/802772272","heartbeat_back_addr":"172.21.15.130:6825/802772272","heartbeat_front_addr":"172.21.15.130:6823/802772272","state":["exists","up"]},{"osd":3,"uuid":"9093e7a3-e43a-4387-bb67-c5f6eff53bae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":929040654},{"type":"v1","addr":"172.21.15.130:6827","nonce":929040654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":929040654},{"type":"v1","addr":"172.21.15.130:6829","nonce":929040654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":929040654},{"type":"v1","addr":"172.21.15.130:6833","nonce":929040654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":929040654},{"type":"v1","addr":"172.21.15.130:6831","nonce":929040654}]},"public_addr":"172.21.15.130:6827/929040654","cluster_addr":"172.21.15.130:6829/929040654","heartbeat_back_addr":"172.21.15.130:6833/929040654","heartbeat_front_addr":"172.21.15.130:6831/929040654","state":["exists","up"]},{"osd":4,"uuid":"872ec63d-c15e-4680-ab71-e1cf974154cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":3400040487},{"type":"v1","addr":"172.21.15.150:6801","nonce":3400040487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":3400040487},{"type":"v1","addr":"172.21.15.150:6803","nonce":3400040487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":3400040487},{"type":"v1","addr":"172.21.15.150:6807","nonce":3400040487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":3400040487},{"type":"v1","addr":"172.21.15.150:6805","nonce":3400040487}]},"public_addr":"172.21.15.150:6801/3400040487","cluster_addr":"172.21.15.150:6803/3400040487","heartbeat_back_addr":"172.21.15.150:6807/3400040487","heartbeat_front_addr":"172.21.15.150:6805/3400040487","state":["exists","up"]},{"osd":5,"uuid":"67066d12-b377-4cc6-93f0-ce22e7fb1de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":3183024588},{"type":"v1","addr":"172.21.15.150:6809","nonce":3183024588}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3183024588},{"type":"v1","addr":"172.21.15.150:6811","nonce":3183024588}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3183024588},{"type":"v1","addr":"172.21.15.150:6815","nonce":3183024588}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3183024588},{"type":"v1","addr":"172.21.15.150:6813","nonce":3183024588}]},"public_addr":"172.21.15.150:6809/3183024588","cluster_addr":"172.21.15.150:6811/3183024588","heartbeat_back_addr":"172.21.15.150:6815/3183024588","heartbeat_front_addr":"172.21.15.150:6813/3183024588","state":["exists","up"]},{"osd":6,"uuid":"8f18c163-a1a4-4016-9848-1d43db8a0c52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":3361358828},{"type":"v1","addr":"172.21.15.150:6817","nonce":3361358828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":3361358828},{"type":"v1","addr":"172.21.15.150:6819","nonce":3361358828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":3361358828},{"type":"v1","addr":"172.21.15.150:6823","nonce":3361358828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":3361358828},{"type":"v1","addr":"172.21.15.150:6821","nonce":3361358828}]},"public_addr":"172.21.15.150:6817/3361358828","cluster_addr":"172.21.15.150:6819/3361358828","heartbeat_back_addr":"172.21.15.150:6823/3361358828","heartbeat_front_addr":"172.21.15.150:6821/3361358828","state":["exists","up"]},{"osd":7,"uuid":"f45d779f-52bd-4994-b35d-61601dda32ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":2168510914},{"type":"v1","addr":"172.21.15.150:6825","nonce":2168510914}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":2168510914},{"type":"v1","addr":"172.21.15.150:6827","nonce":2168510914}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":2168510914},{"type":"v1","addr":"172.21.15.150:6831","nonce":2168510914}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":2168510914},{"type":"v1","addr":"172.21.15.150:6829","nonce":2168510914}]},"public_addr":"172.21.15.150:6825/2168510914","cluster_addr":"172.21.15.150:6827/2168510914","heartbeat_back_addr":"172.21.15.150:6831/2168510914","heartbeat_front_addr":"172.21.15.150:6829/2168510914","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:07:00.637423+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:07:24.770703+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:07:49.217681+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:08:13.854620+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:08:35.673654+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:08:59.692065+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:09:23.718847+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T21:09:48.470551+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/4195343556":"2024-04-23T21:10:09.086654+0000","172.21.15.130:0/1551796354":"2024-04-23T21:10:09.086654+0000","172.21.15.130:0/3442044605":"2024-04-23T21:05:41.702451+0000","172.21.15.130:6800/4050722828":"2024-04-23T21:10:09.086654+0000","172.21.15.130:6801/1754652716":"2024-04-23T21:05:41.702451+0000","172.21.15.130:6800/1814259082":"2024-04-23T21:05:19.464652+0000","172.21.15.130:6801/4050722828":"2024-04-23T21:10:09.086654+0000","172.21.15.130:0/2290497716":"2024-04-23T21:05:41.702451+0000","172.21.15.130:0/636209570":"2024-04-23T21:05:19.464652+0000","172.21.15.130:0/2397194687":"2024-04-23T21:05:41.702451+0000","172.21.15.130:6801/1814259082":"2024-04-23T21:05:19.464652+0000","172.21.15.130:0/2845297619":"2024-04-23T21:05:19.464652+0000","172.21.15.130:0/4252030715":"2024-04-23T21:10:09.086654+0000","172.21.15.130:0/3720321995":"2024-04-23T21:10:09.086654+0000","172.21.15.130:6800/1754652716":"2024-04-23T21:05:41.702451+0000","172.21.15.130:0/4177177338":"2024-04-23T21:05:19.464652+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T21:10:36.861 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:10:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:10:37.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:37 smithi130 ceph-mon[26394]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:37.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:37 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3846208088' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T21:10:37.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:37 smithi130 ceph-mon[21867]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:37.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:37 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3846208088' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T21:10:37.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:37 smithi150 ceph-mon[22676]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:37.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:37 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3846208088' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T21:10:37.396 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-22T21:10:37.396 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-22T21:10:37.396 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-22T21:10:37.397 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-22T21:10:37.397 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-22T21:10:37.397 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-22T21:10:37.398 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-22T21:10:37.398 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-22T21:10:37.800 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:37.801 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:37.802 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:37.813 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:37.814 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:37.826 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:37.832 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:37.832 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:38.360 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:37 smithi150 bash[38020]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-22T21:10:38.361 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 bash[38020]: Writing manifest to image destination 2024-04-22T21:10:38.738 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 podman[38020]: 2024-04-22 21:10:38.377477486 +0000 UTC m=+3.452955124 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-22T21:10:38.738 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 podman[38020]: 2024-04-22 21:10:38.475941728 +0000 UTC m=+3.551419424 container create aafb8379315730f2eed5b3877618269314d6c94695bd78fbde37c6a6ceeb9cb5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:10:38.738 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 podman[38020]: 2024-04-22 21:10:38.661238459 +0000 UTC m=+3.736716096 container init aafb8379315730f2eed5b3877618269314d6c94695bd78fbde37c6a6ceeb9cb5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:10:38.738 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 podman[38020]: 2024-04-22 21:10:38.664377933 +0000 UTC m=+3.739855576 container start aafb8379315730f2eed5b3877618269314d6c94695bd78fbde37c6a6ceeb9cb5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:10:38.738 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.669Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-22T21:10:38.739 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.669Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-22T21:10:38.739 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-22T21:10:38.739 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T21:10:38.739 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-22T21:10:38.739 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=arp 2024-04-22T21:10:38.739 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-22T21:10:38.739 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-22T21:10:38.739 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-22T21:10:38.739 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-22T21:10:38.739 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-22T21:10:38.739 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-22T21:10:38.739 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-22T21:10:38.739 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=edac 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-22T21:10:38.740 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-22T21:10:38.741 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-22T21:10:38.741 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-22T21:10:38.741 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=os 2024-04-22T21:10:38.741 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-22T21:10:38.741 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-22T21:10:38.741 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-22T21:10:38.741 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-22T21:10:38.741 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-22T21:10:38.741 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-22T21:10:38.742 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=stat 2024-04-22T21:10:38.742 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-22T21:10:38.742 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-22T21:10:38.742 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-22T21:10:38.742 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=time 2024-04-22T21:10:38.742 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-22T21:10:38.742 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=uname 2024-04-22T21:10:38.742 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-22T21:10:38.742 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-22T21:10:38.742 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-22T21:10:38.742 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.670Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-22T21:10:38.742 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[38070]: ts=2024-04-22T21:10:38.671Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-22T21:10:38.742 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 bash[38020]: aafb8379315730f2eed5b3877618269314d6c94695bd78fbde37c6a6ceeb9cb5 2024-04-22T21:10:39.059 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:10:38 smithi150 systemd[1]: Started Ceph node-exporter.b for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:10:39.130 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:39 smithi130 ceph-mon[21867]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:39.130 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:39 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:39.130 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:39 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:10:39.130 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:39 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:10:39.131 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:39 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:10:39.131 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:39 smithi130 ceph-mon[26394]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:39.131 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:39 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:39.131 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:39 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:10:39.131 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:39 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:10:39.131 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:39 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:10:39.326 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:39 smithi150 ceph-mon[22676]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:39.326 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:39 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:39.326 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:39 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:10:39.326 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:39 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:10:39.326 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:39 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:10:40.196 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:10:39] "GET /metrics HTTP/1.1" 200 191504 "" "Prometheus/2.33.4" 2024-04-22T21:10:41.306 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:41 smithi150 ceph-mon[22676]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:41.307 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:41 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:41.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:41 smithi130 ceph-mon[21867]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:41.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:41 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:41.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:41 smithi130 ceph-mon[26394]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:41.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:41 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:42.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:42 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:42.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:42 smithi130 ceph-mon[21867]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:42.905 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:42 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:42.905 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:42 smithi130 ceph-mon[26394]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:43.110 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:42 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:10:43.110 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:42 smithi150 ceph-mon[22676]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:45.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:45 smithi130 ceph-mon[26394]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:45.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:44 smithi130 ceph-mon[21867]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:45.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:44 smithi150 ceph-mon[22676]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:46.861 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:46 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:10:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:10:47.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:47 smithi130 ceph-mon[21867]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:47.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:47 smithi130 ceph-mon[26394]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:47.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:47 smithi150 ceph-mon[22676]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:48.907 INFO:teuthology.orchestra.run.smithi130.stdout:34359738415 2024-04-22T21:10:48.908 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-22T21:10:49.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:49 smithi130 ceph-mon[21867]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:49.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:49 smithi130 ceph-mon[26394]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:49.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:49 smithi150 ceph-mon[22676]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:50.023 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:10:49] "GET /metrics HTTP/1.1" 200 191491 "" "Prometheus/2.33.4" 2024-04-22T21:10:51.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:51 smithi150 ceph-mon[22676]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:51.474 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:51 smithi130 ceph-mon[26394]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:51.475 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:51 smithi130 ceph-mon[21867]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:51.502 INFO:teuthology.orchestra.run.smithi130.stdout:55834574891 2024-04-22T21:10:51.502 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-22T21:10:52.705 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:52.721 INFO:teuthology.orchestra.run.smithi130.stdout:77309411365 2024-04-22T21:10:52.722 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-22T21:10:53.134 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:53 smithi130 ceph-mon[26394]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:53.134 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:53 smithi130 ceph-mon[21867]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:53.157 INFO:teuthology.orchestra.run.smithi130.stdout:197568495635 2024-04-22T21:10:53.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-22T21:10:53.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:53 smithi150 ceph-mon[22676]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:54.795 INFO:teuthology.orchestra.run.smithi130.stdout:163208757272 2024-04-22T21:10:54.796 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-22T21:10:55.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:55 smithi130 ceph-mon[26394]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:55.034 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:55 smithi130 ceph-mon[21867]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:55.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:55 smithi150 ceph-mon[22676]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:55.464 INFO:teuthology.orchestra.run.smithi130.stdout:128849018907 2024-04-22T21:10:55.464 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-22T21:10:56.404 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:56.405 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:56.860 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:10:56 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:10:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:10:56.912 INFO:teuthology.orchestra.run.smithi130.stdout:231928233999 2024-04-22T21:10:56.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-22T21:10:57.082 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:57.163 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:57 smithi130 ceph-mon[26394]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:57.163 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:57 smithi130 ceph-mon[21867]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:57.311 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:57.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:57 smithi150 ceph-mon[22676]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:57.811 INFO:teuthology.orchestra.run.smithi130.stdout:107374182435 2024-04-22T21:10:57.811 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-22T21:10:57.853 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:10:59.190 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:10:59 smithi130 ceph-mon[21867]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:59.190 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:10:59 smithi130 ceph-mon[26394]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:10:59.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:10:59 smithi150 ceph-mon[22676]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:00.307 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:10:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:10:59] "GET /metrics HTTP/1.1" 200 191483 "" "Prometheus/2.33.4" 2024-04-22T21:11:01.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:01 smithi130 ceph-mon[21867]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:01.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:01 smithi130 ceph-mon[26394]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:01.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:01 smithi150 ceph-mon[22676]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:02.582 INFO:teuthology.orchestra.run.smithi130.stdout:34359738418 2024-04-22T21:11:03.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:03 smithi130 ceph-mon[21867]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:03.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:03 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3241216006' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-22T21:11:03.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:03 smithi130 ceph-mon[26394]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:03.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:03 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3241216006' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-22T21:11:03.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:03 smithi150 ceph-mon[22676]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:03.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:03 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3241216006' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-22T21:11:03.745 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738415 got 34359738418 for osd.0 2024-04-22T21:11:03.745 DEBUG:teuthology.parallel:result is None 2024-04-22T21:11:04.137 INFO:teuthology.orchestra.run.smithi130.stdout:163208757274 2024-04-22T21:11:05.285 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:11:05.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:05 smithi130 ceph-mon[21867]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:05.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:05 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4208131354' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-22T21:11:05.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:05 smithi130 ceph-mon[26394]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:05.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:05 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4208131354' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-22T21:11:05.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:05 smithi150 ceph-mon[22676]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:05.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:05 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4208131354' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-22T21:11:06.503 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757272 got 163208757274 for osd.5 2024-04-22T21:11:06.504 DEBUG:teuthology.parallel:result is None 2024-04-22T21:11:06.582 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:06 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:06.582 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:06 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:06.582 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:06 smithi130 ceph-mon[26394]: Deploying daemon alertmanager.a on smithi130 2024-04-22T21:11:06.582 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:06 smithi130 ceph-mon[26394]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:06.582 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:06 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:06.582 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:06 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:06.583 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:06 smithi130 ceph-mon[21867]: Deploying daemon alertmanager.a on smithi130 2024-04-22T21:11:06.583 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:06 smithi130 ceph-mon[21867]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:06.610 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:11:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:11:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:11:06.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:06 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:06.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:06 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:06.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:06 smithi150 ceph-mon[22676]: Deploying daemon alertmanager.a on smithi130 2024-04-22T21:11:06.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:06 smithi150 ceph-mon[22676]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:06.665 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:11:06.862 INFO:teuthology.orchestra.run.smithi130.stdout:128849018911 2024-04-22T21:11:07.499 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:07 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3409530667' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-22T21:11:07.499 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:07 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3409530667' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-22T21:11:07.499 INFO:teuthology.orchestra.run.smithi130.stdout:77309411370 2024-04-22T21:11:07.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:07 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3409530667' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-22T21:11:08.143 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411370 for osd.2 2024-04-22T21:11:08.143 DEBUG:teuthology.parallel:result is None 2024-04-22T21:11:08.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:08 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4275010451' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-22T21:11:08.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:08 smithi130 ceph-mon[21867]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:08.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:08 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4275010451' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-22T21:11:08.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:08 smithi130 ceph-mon[26394]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:08.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:08 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4275010451' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-22T21:11:08.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:08 smithi150 ceph-mon[22676]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:08.682 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018907 got 128849018911 for osd.4 2024-04-22T21:11:08.682 DEBUG:teuthology.parallel:result is None 2024-04-22T21:11:09.273 INFO:teuthology.orchestra.run.smithi130.stdout:55834574895 2024-04-22T21:11:10.182 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:09 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2647723335' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-22T21:11:10.182 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:11:09 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:11:09] "GET /metrics HTTP/1.1" 200 191483 "" "Prometheus/2.33.4" 2024-04-22T21:11:10.183 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:09 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2647723335' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-22T21:11:10.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:09 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2647723335' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-22T21:11:10.815 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574891 got 55834574895 for osd.1 2024-04-22T21:11:10.815 DEBUG:teuthology.parallel:result is None 2024-04-22T21:11:11.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:11 smithi130 ceph-mon[26394]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:11.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:11:11.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:11:11.057 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:11 smithi130 ceph-mon[21867]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:11.057 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:11:11.057 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:11:11.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:11 smithi150 ceph-mon[22676]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:11.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:11:11.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:11:11.991 INFO:teuthology.orchestra.run.smithi130.stdout:197568495639 2024-04-22T21:11:12.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:12 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/826297639' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-22T21:11:12.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:12 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/826297639' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-22T21:11:12.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:12 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/826297639' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-22T21:11:13.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:13 smithi130 ceph-mon[21867]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:13.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:13 smithi130 ceph-mon[26394]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:13.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:13 smithi150 ceph-mon[22676]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:14.328 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495635 got 197568495639 for osd.6 2024-04-22T21:11:14.328 DEBUG:teuthology.parallel:result is None 2024-04-22T21:11:15.117 INFO:teuthology.orchestra.run.smithi130.stdout:231928234003 2024-04-22T21:11:15.117 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:15 smithi130 ceph-mon[21867]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:15.118 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:15 smithi130 ceph-mon[26394]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:15.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:15 smithi150 ceph-mon[22676]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:16.172 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:16 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2963383227' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-22T21:11:16.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:16 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2963383227' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-22T21:11:16.197 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233999 got 231928234003 for osd.7 2024-04-22T21:11:16.197 DEBUG:teuthology.parallel:result is None 2024-04-22T21:11:16.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:16 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2963383227' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-22T21:11:16.588 INFO:teuthology.orchestra.run.smithi130.stdout:107374182438 2024-04-22T21:11:16.860 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:11:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:11:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:11:17.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:17 smithi130 ceph-mon[21867]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:17.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:17 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/866316517' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-22T21:11:17.156 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:17 smithi130 ceph-mon[26394]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:17.156 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:17 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/866316517' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-22T21:11:17.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:17 smithi150 ceph-mon[22676]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:17.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:17 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/866316517' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-22T21:11:17.470 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182435 got 107374182438 for osd.3 2024-04-22T21:11:17.470 DEBUG:teuthology.parallel:result is None 2024-04-22T21:11:17.470 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-22T21:11:17.471 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-22T21:11:18.855 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:11:19.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:19 smithi150 ceph-mon[22676]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:19.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:19 smithi130 ceph-mon[21867]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:19.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:19 smithi130 ceph-mon[26394]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:20.056 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:11:19 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:11:19] "GET /metrics HTTP/1.1" 200 191492 "" "Prometheus/2.33.4" 2024-04-22T21:11:21.181 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:21 smithi130 ceph-mon[21867]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:21.181 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:21 smithi130 ceph-mon[26394]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:21.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:21 smithi150 ceph-mon[22676]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:21.377 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:11:21.378 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-04-22T21:11:21.556 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:21 smithi130 podman[44691]: 2024-04-22 21:11:21.431412767 +0000 UTC m=+0.720217783 container create a2cc30afc0f2dcc2837498aff8aff4c19d9243c1109b35e763894f7f221670b3 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:21.852 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:21 smithi130 podman[44691]: 2024-04-22 21:11:21.759267876 +0000 UTC m=+1.048072903 container init a2cc30afc0f2dcc2837498aff8aff4c19d9243c1109b35e763894f7f221670b3 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:21.852 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:21 smithi130 podman[44691]: 2024-04-22 21:11:21.761928378 +0000 UTC m=+1.050733404 container start a2cc30afc0f2dcc2837498aff8aff4c19d9243c1109b35e763894f7f221670b3 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:21.852 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:21 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[44745]: level=info ts=2024-04-22T21:11:21.772Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-22T21:11:21.852 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:21 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[44745]: level=info ts=2024-04-22T21:11:21.772Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-22T21:11:21.852 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:21 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[44745]: level=info ts=2024-04-22T21:11:21.773Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.130 port=9094 2024-04-22T21:11:21.852 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:21 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[44745]: level=info ts=2024-04-22T21:11:21.774Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-22T21:11:21.852 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:21 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[44745]: level=info ts=2024-04-22T21:11:21.799Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T21:11:21.852 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:21 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[44745]: level=info ts=2024-04-22T21:11:21.800Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T21:11:21.852 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:21 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[44745]: level=info ts=2024-04-22T21:11:21.802Z caller=main.go:518 msg=Listening address=:9093 2024-04-22T21:11:21.852 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:21 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[44745]: level=info ts=2024-04-22T21:11:21.802Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-22T21:11:21.852 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:21 smithi130 bash[44691]: a2cc30afc0f2dcc2837498aff8aff4c19d9243c1109b35e763894f7f221670b3 2024-04-22T21:11:22.241 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:21 smithi130 systemd[1]: Started Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:11:22.563 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-04-22T21:11:20.013781+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48840,"kb_used_data":6088,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912376,"statfs":{"total":767960285184,"available":767910273024,"internally_reserved":0,"allocated":6234112,"data_stored":3488394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001798"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":266,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-22T21:10:10.079883+0000","last_change":"2024-04-22T21:09:54.283256+0000","last_active":"2024-04-22T21:10:10.079883+0000","last_peered":"2024-04-22T21:10:10.079883+0000","last_clean":"2024-04-22T21:10:10.079883+0000","last_became_active":"2024-04-22T21:09:54.282728+0000","last_became_peered":"2024-04-22T21:09:54.282728+0000","last_unstale":"2024-04-22T21:10:10.079883+0000","last_undegraded":"2024-04-22T21:10:10.079883+0000","last_fullsized":"2024-04-22T21:10:10.079883+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-22T21:07:52.067024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-22T21:07:52.067024+0000","last_clean_scrub_stamp":"2024-04-22T21:07:52.067024+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-24T04:33:05.242782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234004,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6084,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739068,"statfs":{"total":95995035648,"available":95988805632,"internally_reserved":0,"allocated":1110016,"data_stored":766504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.59599999999999997}]},{"osd":1,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.67200000000000004}]},{"osd":2,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.71999999999999997}]},{"osd":3,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.65000000000000002}]},{"osd":4,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.622}]},{"osd":5,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.751},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.73999999999999999}]},{"osd":6,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.66500000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495641,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6144,"kb_used_data":1080,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739008,"statfs":{"total":95995035648,"available":95988744192,"internally_reserved":0,"allocated":1105920,"data_stored":766181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:10:30 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.503}]},{"osd":1,"last update":"Mon Apr 22 21:10:30 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.55200000000000005}]},{"osd":2,"last update":"Mon Apr 22 21:10:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.1879999999999999,"5min":1.1879999999999999,"15min":1.1879999999999999},"last":0.45600000000000002}]},{"osd":3,"last update":"Mon Apr 22 21:10:30 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.60299999999999998}]},{"osd":4,"last update":"Mon Apr 22 21:10:30 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.5660000000000001,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":0.58399999999999996}]},{"osd":5,"last update":"Mon Apr 22 21:10:30 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.214,"5min":1.214,"15min":1.214},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.627,"5min":1.627,"15min":1.627},"last":0.63300000000000001}]},{"osd":7,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.61499999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6336,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738816,"statfs":{"total":95995035648,"available":95988547584,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:10:37 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.34899999999999998,"15min":0.33000000000000002},"min":{"1min":0.23699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.53200000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.37,"15min":0.34799999999999998},"min":{"1min":0.28199999999999997,"5min":0.22,"15min":0.22},"max":{"1min":0.52000000000000002,"5min":0.624,"15min":0.624},"last":0.33400000000000002}]},{"osd":2,"last update":"Mon Apr 22 21:10:59 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.39900000000000002,"15min":0.39500000000000002},"min":{"1min":0.25,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.59199999999999997,"5min":0.629,"15min":0.629},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.39300000000000002,"15min":0.38500000000000001},"min":{"1min":0.26400000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.54900000000000004,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.35099999999999998}]},{"osd":3,"last update":"Mon Apr 22 21:10:19 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.46300000000000002,"15min":0.46500000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.309},{"interface":"front","average":{"1min":0.436,"5min":0.45800000000000002,"15min":0.46200000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.28599999999999998}]},{"osd":4,"last update":"Mon Apr 22 21:10:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.71699999999999997,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.59399999999999997,"15min":0.58999999999999997},"min":{"1min":0.47999999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.72299999999999998,"5min":0.871,"15min":0.871},"last":0.72899999999999998}]},{"osd":5,"last update":"Mon Apr 22 21:11:07 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.58599999999999997,"15min":0.57899999999999996},"min":{"1min":0.5,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.78400000000000003,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.58199999999999996,"15min":0.57999999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.67200000000000004}]},{"osd":6,"last update":"Mon Apr 22 21:10:26 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.66000000000000003}]},{"osd":7,"last update":"Mon Apr 22 21:10:50 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.70399999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6920,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738232,"statfs":{"total":95995035648,"available":95987949568,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Apr 22 21:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.34300000000000003,"15min":0.32100000000000001},"min":{"1min":0.251,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.55500000000000005,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.35899999999999999,"15min":0.33600000000000002},"min":{"1min":0.312,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.55200000000000005,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.47899999999999998}]},{"osd":2,"last update":"Mon Apr 22 21:10:56 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.38900000000000001,"15min":0.374},"min":{"1min":0.29299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.49399999999999999,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.376,"15min":0.36799999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.53200000000000003,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.46400000000000002}]},{"osd":3,"last update":"Mon Apr 22 21:10:20 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.41199999999999998,"15min":0.40999999999999998},"min":{"1min":0.27400000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.55300000000000005,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45900000000000002,"15min":0.46100000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.43099999999999999}]},{"osd":4,"last update":"Mon Apr 22 21:10:41 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.53800000000000003,"15min":0.52800000000000002},"min":{"1min":0.36399999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.56899999999999995,"15min":0.56100000000000005},"min":{"1min":0.44900000000000001,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.68899999999999995}]},{"osd":5,"last update":"Mon Apr 22 21:11:03 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.55900000000000005,"15min":0.55300000000000005},"min":{"1min":0.41499999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.78900000000000003,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.63700000000000001}]},{"osd":6,"last update":"Mon Apr 22 21:10:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.623}]},{"osd":7,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.65500000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411372,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6272,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738880,"statfs":{"total":95995035648,"available":95988613120,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:10:58 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.38100000000000001,"15min":0.377},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.56000000000000005,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.38300000000000001,"15min":0.375},"min":{"1min":0.26900000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.54300000000000004,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.36399999999999999}]},{"osd":1,"last update":"Mon Apr 22 21:10:58 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.38500000000000001,"15min":0.38100000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.53200000000000003,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.40400000000000003,"15min":0.39200000000000002},"min":{"1min":0.30099999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.58899999999999997,"5min":0.623,"15min":0.623},"last":0.34000000000000002}]},{"osd":3,"last update":"Mon Apr 22 21:10:16 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.44900000000000001,"15min":0.45000000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.435,"15min":0.438},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.60399999999999998,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.27300000000000002}]},{"osd":4,"last update":"Mon Apr 22 21:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.76800000000000002,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.59399999999999997,"15min":0.58999999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.78100000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.63400000000000001}]},{"osd":5,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58899999999999997,"15min":0.58999999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.81200000000000006,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.39300000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.64900000000000002}]},{"osd":6,"last update":"Mon Apr 22 21:10:26 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.59399999999999997}]},{"osd":7,"last update":"Mon Apr 22 21:10:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.60499999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182439,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:10:19 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.40100000000000002,"15min":0.39300000000000002},"min":{"1min":0.309,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.623,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.44700000000000001,"15min":0.443},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.35099999999999998}]},{"osd":1,"last update":"Mon Apr 22 21:10:19 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.41499999999999998,"15min":0.41299999999999998},"min":{"1min":0.28599999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.42599999999999999,"15min":0.41999999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.39600000000000002}]},{"osd":2,"last update":"Mon Apr 22 21:10:19 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.44,"15min":0.436},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.441,"5min":0.41699999999999998,"15min":0.41299999999999998},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.34000000000000002}]},{"osd":4,"last update":"Mon Apr 22 21:10:40 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55500000000000005,"15min":0.55200000000000005},"min":{"1min":0.39900000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.54100000000000004,"15min":0.53300000000000003},"min":{"1min":0.40500000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.53400000000000003}]},{"osd":5,"last update":"Mon Apr 22 21:11:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.58999999999999997,"15min":0.59199999999999997},"min":{"1min":0.42699999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.499},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.59499999999999997,"15min":0.59799999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.80800000000000005,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.56699999999999995}]},{"osd":6,"last update":"Mon Apr 22 21:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.48899999999999999}]},{"osd":7,"last update":"Mon Apr 22 21:10:52 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.55400000000000005}]}]},{"osd":4,"up_from":30,"seq":128849018913,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:10:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.38400000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.50900000000000001,"15min":0.50700000000000001},"min":{"1min":0.39000000000000001,"5min":0.373,"15min":0.373},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.63900000000000001}]},{"osd":1,"last update":"Mon Apr 22 21:10:38 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.51400000000000001,"15min":0.51600000000000001},"min":{"1min":0.38900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.51700000000000002,"15min":0.51400000000000001},"min":{"1min":0.36299999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.72899999999999998}]},{"osd":2,"last update":"Mon Apr 22 21:10:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51000000000000001,"15min":0.50600000000000001},"min":{"1min":0.38900000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.53300000000000003,"15min":0.53400000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.63400000000000001,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.66000000000000003}]},{"osd":3,"last update":"Mon Apr 22 21:10:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.505,"15min":0.501},"min":{"1min":0.371,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.67900000000000005,"15min":0.70099999999999996},"min":{"1min":0.35199999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.79500000000000004,"5min":4.5019999999999998,"15min":4.5019999999999998},"last":0.70599999999999996}]},{"osd":5,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.38100000000000001,"15min":0.36599999999999999},"min":{"1min":0.29899999999999999,"5min":0.221,"15min":0.221},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.502,"5min":0.40699999999999997,"15min":0.39100000000000001},"min":{"1min":0.28799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.57499999999999996}]},{"osd":6,"last update":"Mon Apr 22 21:10:26 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.60599999999999998}]},{"osd":7,"last update":"Mon Apr 22 21:10:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.626},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.58999999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757277,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5692,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739460,"statfs":{"total":95995035648,"available":95989207040,"internally_reserved":0,"allocated":577536,"data_stored":237576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.51600000000000001,"15min":0.51000000000000001},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.376,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.66100000000000003,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.51800000000000002}]},{"osd":1,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.52900000000000003,"15min":0.52600000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.70399999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.505},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.63500000000000001,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.44800000000000001}]},{"osd":2,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.52200000000000002,"15min":0.52500000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.60899999999999999,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.434},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.54200000000000004,"15min":0.53700000000000003},"min":{"1min":0.434,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.73299999999999998,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.57199999999999995}]},{"osd":3,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56799999999999995,"15min":0.56999999999999995},"min":{"1min":0.41599999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.68600000000000005,"5min":0.879,"15min":0.879},"last":0.497},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58999999999999997,"15min":0.58799999999999997},"min":{"1min":0.437,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.70399999999999996,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.42499999999999999}]},{"osd":4,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.45300000000000001,"15min":0.44900000000000001},"min":{"1min":0.33900000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.65200000000000002,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.373},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.46300000000000002,"15min":0.46000000000000002},"min":{"1min":0.33900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.66800000000000004,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.46899999999999997}]},{"osd":6,"last update":"Mon Apr 22 21:10:29 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.48099999999999998}]},{"osd":7,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.49099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-22T21:11:22.564 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-22T21:11:22.779 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:11:23.234 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[21867]: from='client.24508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T21:11:23.234 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:23.234 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:23.234 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:23.235 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T21:11:23.235 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T21:11:23.235 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:23.235 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[21867]: Deploying daemon grafana.a on smithi150 2024-04-22T21:11:23.235 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[21867]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:23.235 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[26394]: from='client.24508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T21:11:23.235 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:23.235 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:23.236 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:23.236 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T21:11:23.236 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[26394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T21:11:23.236 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:23.236 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[26394]: Deploying daemon grafana.a on smithi150 2024-04-22T21:11:23.236 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:22 smithi130 ceph-mon[26394]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:23.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:22 smithi150 ceph-mon[22676]: from='client.24508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T21:11:23.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:22 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:23.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:22 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:23.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:22 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:23.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:22 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T21:11:23.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:22 smithi150 ceph-mon[22676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T21:11:23.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:22 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:23.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:22 smithi150 ceph-mon[22676]: Deploying daemon grafana.a on smithi150 2024-04-22T21:11:23.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:22 smithi150 ceph-mon[22676]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:23.962 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:23 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[44745]: level=info ts=2024-04-22T21:11:23.775Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000447352s 2024-04-22T21:11:24.662 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:11:24.662 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-04-22T21:11:25.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:25 smithi130 ceph-mon[21867]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:25.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:25 smithi130 ceph-mon[26394]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:25.354 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":40,"stamp":"2024-04-22T21:11:24.014334+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48840,"kb_used_data":6088,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912376,"statfs":{"total":767960285184,"available":767910273024,"internally_reserved":0,"allocated":6234112,"data_stored":3488394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001776"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":266,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-22T21:10:10.079883+0000","last_change":"2024-04-22T21:09:54.283256+0000","last_active":"2024-04-22T21:10:10.079883+0000","last_peered":"2024-04-22T21:10:10.079883+0000","last_clean":"2024-04-22T21:10:10.079883+0000","last_became_active":"2024-04-22T21:09:54.282728+0000","last_became_peered":"2024-04-22T21:09:54.282728+0000","last_unstale":"2024-04-22T21:10:10.079883+0000","last_undegraded":"2024-04-22T21:10:10.079883+0000","last_fullsized":"2024-04-22T21:10:10.079883+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-22T21:07:52.067024+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-22T21:07:52.067024+0000","last_clean_scrub_stamp":"2024-04-22T21:07:52.067024+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-24T04:33:05.242782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234005,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6084,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739068,"statfs":{"total":95995035648,"available":95988805632,"internally_reserved":0,"allocated":1110016,"data_stored":766504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.68700000000000006}]},{"osd":1,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.72699999999999998}]},{"osd":2,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.71099999999999997}]},{"osd":3,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.74399999999999999}]},{"osd":4,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.67200000000000004}]},{"osd":5,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.77500000000000002}]},{"osd":6,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.752},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.70299999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495642,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6144,"kb_used_data":1080,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739008,"statfs":{"total":95995035648,"available":95988744192,"internally_reserved":0,"allocated":1105920,"data_stored":766181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:10:30 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.51500000000000001}]},{"osd":1,"last update":"Mon Apr 22 21:10:30 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.497}]},{"osd":2,"last update":"Mon Apr 22 21:10:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.1879999999999999,"5min":1.1879999999999999,"15min":1.1879999999999999},"last":0.59499999999999997}]},{"osd":3,"last update":"Mon Apr 22 21:10:30 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.595,"5min":1.595,"15min":1.595},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.70399999999999996}]},{"osd":4,"last update":"Mon Apr 22 21:10:30 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.5660000000000001,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":0.66100000000000003}]},{"osd":5,"last update":"Mon Apr 22 21:10:30 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.214,"5min":1.214,"15min":1.214},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.627,"5min":1.627,"15min":1.627},"last":0.68200000000000005}]},{"osd":7,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.76900000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6336,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738816,"statfs":{"total":95995035648,"available":95988547584,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:10:37 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.34899999999999998,"15min":0.33000000000000002},"min":{"1min":0.23699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.53200000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.37,"15min":0.34799999999999998},"min":{"1min":0.28199999999999997,"5min":0.22,"15min":0.22},"max":{"1min":0.52000000000000002,"5min":0.624,"15min":0.624},"last":0.36199999999999999}]},{"osd":2,"last update":"Mon Apr 22 21:10:59 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.39900000000000002,"15min":0.39500000000000002},"min":{"1min":0.25,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.59199999999999997,"5min":0.629,"15min":0.629},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.39300000000000002,"15min":0.38500000000000001},"min":{"1min":0.26400000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.54900000000000004,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.308}]},{"osd":3,"last update":"Mon Apr 22 21:10:19 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.46300000000000002,"15min":0.46500000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.436,"5min":0.45800000000000002,"15min":0.46200000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.34699999999999998}]},{"osd":4,"last update":"Mon Apr 22 21:10:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.71699999999999997,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.59399999999999997,"15min":0.58999999999999997},"min":{"1min":0.47999999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.72299999999999998,"5min":0.871,"15min":0.871},"last":0.64800000000000002}]},{"osd":5,"last update":"Mon Apr 22 21:11:07 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.58599999999999997,"15min":0.57899999999999996},"min":{"1min":0.5,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.78400000000000003,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.58199999999999996,"15min":0.57999999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.68700000000000006}]},{"osd":6,"last update":"Mon Apr 22 21:10:26 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.628}]},{"osd":7,"last update":"Mon Apr 22 21:10:50 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.63600000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6920,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738232,"statfs":{"total":95995035648,"available":95987949568,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Apr 22 21:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.34300000000000003,"15min":0.32100000000000001},"min":{"1min":0.251,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.55500000000000005,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.35899999999999999,"15min":0.33600000000000002},"min":{"1min":0.312,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.55200000000000005,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.40100000000000002}]},{"osd":2,"last update":"Mon Apr 22 21:10:56 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.38900000000000001,"15min":0.374},"min":{"1min":0.29299999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.49399999999999999,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.376,"15min":0.36799999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.53200000000000003,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.42699999999999999}]},{"osd":3,"last update":"Mon Apr 22 21:10:20 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.41199999999999998,"15min":0.40999999999999998},"min":{"1min":0.27400000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.55300000000000005,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45900000000000002,"15min":0.46100000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.46000000000000002}]},{"osd":4,"last update":"Mon Apr 22 21:10:41 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.53800000000000003,"15min":0.52800000000000002},"min":{"1min":0.36399999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.56899999999999995,"15min":0.56100000000000005},"min":{"1min":0.44900000000000001,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.72899999999999998}]},{"osd":5,"last update":"Mon Apr 22 21:11:03 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.55900000000000005,"15min":0.55300000000000005},"min":{"1min":0.41499999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.78900000000000003,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.78800000000000003}]},{"osd":6,"last update":"Mon Apr 22 21:10:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.71899999999999997}]},{"osd":7,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.76100000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6272,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738880,"statfs":{"total":95995035648,"available":95988613120,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:10:58 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.38100000000000001,"15min":0.377},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.56000000000000005,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.38300000000000001,"15min":0.375},"min":{"1min":0.26900000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.54300000000000004,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.35899999999999999}]},{"osd":1,"last update":"Mon Apr 22 21:10:58 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.38500000000000001,"15min":0.38100000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.53200000000000003,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.40400000000000003,"15min":0.39200000000000002},"min":{"1min":0.30099999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.58899999999999997,"5min":0.623,"15min":0.623},"last":0.42199999999999999}]},{"osd":3,"last update":"Mon Apr 22 21:11:17 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.439,"15min":0.44700000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.57399999999999995,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.42399999999999999,"15min":0.434},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.55900000000000005,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.38700000000000001}]},{"osd":4,"last update":"Mon Apr 22 21:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.76800000000000002,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.59399999999999997,"15min":0.58999999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.78100000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.59799999999999998}]},{"osd":5,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58899999999999997,"15min":0.58999999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.81200000000000006,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.505},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.39300000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.55000000000000004}]},{"osd":6,"last update":"Mon Apr 22 21:10:26 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.52700000000000002}]},{"osd":7,"last update":"Mon Apr 22 21:10:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.64300000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182440,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:11:20 2024","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.52500000000000002,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.436,"15min":0.439},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.55000000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.29099999999999998}]},{"osd":1,"last update":"Mon Apr 22 21:11:20 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.40400000000000003,"15min":0.40899999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.41199999999999998,"15min":0.41499999999999998},"min":{"1min":0.23699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.51100000000000001,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.35099999999999998}]},{"osd":2,"last update":"Mon Apr 22 21:11:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.42699999999999999,"15min":0.432},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.47199999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.40600000000000003,"15min":0.40899999999999997},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.57899999999999996,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.25800000000000001}]},{"osd":4,"last update":"Mon Apr 22 21:10:40 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55500000000000005,"15min":0.55200000000000005},"min":{"1min":0.39900000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.497},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.54100000000000004,"15min":0.53300000000000003},"min":{"1min":0.40500000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.62}]},{"osd":5,"last update":"Mon Apr 22 21:11:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.58999999999999997,"15min":0.59199999999999997},"min":{"1min":0.42699999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.59499999999999997,"15min":0.59799999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.80800000000000005,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.51000000000000001}]},{"osd":6,"last update":"Mon Apr 22 21:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.61399999999999999}]},{"osd":7,"last update":"Mon Apr 22 21:10:52 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.59299999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018914,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:10:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.38400000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.50900000000000001,"15min":0.50700000000000001},"min":{"1min":0.39000000000000001,"5min":0.373,"15min":0.373},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.749}]},{"osd":1,"last update":"Mon Apr 22 21:10:38 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.51400000000000001,"15min":0.51600000000000001},"min":{"1min":0.38900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.51700000000000002,"15min":0.51400000000000001},"min":{"1min":0.36299999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.64600000000000002}]},{"osd":2,"last update":"Mon Apr 22 21:10:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51000000000000001,"15min":0.50600000000000001},"min":{"1min":0.38900000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.53300000000000003,"15min":0.53400000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.63400000000000001,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.63400000000000001}]},{"osd":3,"last update":"Mon Apr 22 21:10:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.505,"15min":0.501},"min":{"1min":0.371,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.67900000000000005,"15min":0.70099999999999996},"min":{"1min":0.35199999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.79500000000000004,"5min":4.5019999999999998,"15min":4.5019999999999998},"last":0.66800000000000004}]},{"osd":5,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.38100000000000001,"15min":0.36599999999999999},"min":{"1min":0.29899999999999999,"5min":0.221,"15min":0.221},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.502,"5min":0.40699999999999997,"15min":0.39100000000000001},"min":{"1min":0.28799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.73099999999999998}]},{"osd":6,"last update":"Mon Apr 22 21:10:26 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.81000000000000005}]},{"osd":7,"last update":"Mon Apr 22 21:10:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.77200000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757278,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5692,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739460,"statfs":{"total":95995035648,"available":95989207040,"internally_reserved":0,"allocated":577536,"data_stored":237576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.51600000000000001,"15min":0.51000000000000001},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.376,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.66100000000000003,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.81899999999999995}]},{"osd":1,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.52900000000000003,"15min":0.52600000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.70399999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.63500000000000001,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.77200000000000002}]},{"osd":2,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.52200000000000002,"15min":0.52500000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.60899999999999999,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.54200000000000004,"15min":0.53700000000000003},"min":{"1min":0.434,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.73299999999999998,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.72399999999999998}]},{"osd":3,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56799999999999995,"15min":0.56999999999999995},"min":{"1min":0.41599999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.68600000000000005,"5min":0.879,"15min":0.879},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58999999999999997,"15min":0.58799999999999997},"min":{"1min":0.437,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.70399999999999996,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.79000000000000004}]},{"osd":4,"last update":"Mon Apr 22 21:11:05 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.45300000000000001,"15min":0.44900000000000001},"min":{"1min":0.33900000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.65200000000000002,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.46300000000000002,"15min":0.46000000000000002},"min":{"1min":0.33900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.66800000000000004,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.51000000000000001}]},{"osd":6,"last update":"Mon Apr 22 21:10:29 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.59099999999999997}]},{"osd":7,"last update":"Mon Apr 22 21:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.49199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-22T21:11:25.356 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-22T21:11:25.356 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-22T21:11:25.356 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-22T21:11:25.356 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-22T21:11:25.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:25 smithi150 ceph-mon[22676]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:25.566 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T21:11:26.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:26 smithi130 ceph-mon[21867]: from='client.24514 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T21:11:26.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:26 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:26.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:26 smithi130 ceph-mon[26394]: from='client.24514 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T21:11:26.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:26 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:26.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:26 smithi150 ceph-mon[22676]: from='client.24514 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T21:11:26.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:26 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:26.860 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:11:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:11:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:11:27.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:27 smithi130 ceph-mon[21867]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:27.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:27 smithi130 ceph-mon[26394]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:27.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:27 smithi150 ceph-mon[22676]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:27.542 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:11:27.543 INFO:teuthology.orchestra.run.smithi130.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-22T21:11:28.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:28 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/100149586' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-22T21:11:28.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:28 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/100149586' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-22T21:11:28.185 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-22T21:11:28.186 INFO:tasks.cephadm:Setup complete, yielding 2024-04-22T21:11:28.186 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T21:11:28.198 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi130.front.sepia.ceph.com 2024-04-22T21:11:28.199 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest false --force' 2024-04-22T21:11:28.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:28 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/100149586' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-22T21:11:29.196 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:29 smithi130 ceph-mon[21867]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:29.197 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:29 smithi130 ceph-mon[26394]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:29.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:29 smithi150 ceph-mon[22676]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:30.306 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:11:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:11:29] "GET /metrics HTTP/1.1" 200 191487 "" "Prometheus/2.33.4" 2024-04-22T21:11:31.075 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T21:11:31.084 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi130.front.sepia.ceph.com 2024-04-22T21:11:31.084 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-04-22T21:11:31.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:31 smithi130 ceph-mon[21867]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:31.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:31 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' 2024-04-22T21:11:31.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:31 smithi130 ceph-mon[26394]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:31.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:31 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' 2024-04-22T21:11:31.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:31 smithi150 ceph-mon[22676]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:31.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:31 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' 2024-04-22T21:11:32.056 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:31 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[44745]: level=info ts=2024-04-22T21:11:31.778Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003784489s 2024-04-22T21:11:33.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:33 smithi130 ceph-mon[21867]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:33.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:33 smithi130 ceph-mon[26394]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:33.562 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:33 smithi150 ceph-mon[22676]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:34.254 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:11:34.255 INFO:teuthology.orchestra.run.smithi130.stdout: "id": "3d6051a8-76ed-49c6-9e41-0bd7c047ed53", 2024-04-22T21:11:34.255 INFO:teuthology.orchestra.run.smithi130.stdout: "name": "r", 2024-04-22T21:11:34.255 INFO:teuthology.orchestra.run.smithi130.stdout: "current_period": "7ee2ba5a-2e63-4c5a-ac66-8ce17d8c5932", 2024-04-22T21:11:34.255 INFO:teuthology.orchestra.run.smithi130.stdout: "epoch": 1 2024-04-22T21:11:34.255 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:11:34.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:34 smithi150 ceph-mon[22676]: osdmap e60: 8 total, 8 up, 8 in 2024-04-22T21:11:34.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:34 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4179729150' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T21:11:34.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:34 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T21:11:34.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:34 smithi150 ceph-mon[22676]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:34.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:34 smithi130 ceph-mon[21867]: osdmap e60: 8 total, 8 up, 8 in 2024-04-22T21:11:34.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:34 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4179729150' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T21:11:34.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:34 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T21:11:34.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:34 smithi130 ceph-mon[21867]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:34.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:34 smithi130 ceph-mon[26394]: osdmap e60: 8 total, 8 up, 8 in 2024-04-22T21:11:34.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:34 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4179729150' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T21:11:34.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:34 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T21:11:34.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:34 smithi130 ceph-mon[26394]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:34.870 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-04-22T21:11:35.021 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-04-22T21:11:35.021 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-04-22T21:11:35.271 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-22T21:11:35.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-mon[22676]: osdmap e61: 8 total, 8 up, 8 in 2024-04-22T21:11:35.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:35.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:11:35.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:11:35.273 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-04-22T21:11:35.273 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-04-22T21:11:35.273 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-04-22T21:11:35.273 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-04-22T21:11:35.273 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-04-22T21:11:35.273 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-04-22T21:11:35.273 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-04-22T21:11:35.273 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-04-22T21:11:35.273 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-04-22T21:11:35.273 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-22T21:11:35.273 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-04-22T21:11:35.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-04-22T21:11:35.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-04-22T21:11:35.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-04-22T21:11:35.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-04-22T21:11:35.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-04-22T21:11:35.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-04-22T21:11:35.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-04-22T21:11:35.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-04-22T21:11:35.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-04-22T21:11:35.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-04-22T21:11:35.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-04-22T21:11:35.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-04-22T21:11:35.275 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-04-22T21:11:35.275 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-04-22T21:11:35.275 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-04-22T21:11:35.275 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-04-22T21:11:35.275 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-04-22T21:11:35.275 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-04-22T21:11:35.275 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-04-22T21:11:35.275 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-04-22T21:11:35.275 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-04-22T21:11:35.275 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-04-22T21:11:35.275 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-04-22T21:11:35.275 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-22T21:11:35.276 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-04-22T21:11:35.276 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-22T21:11:35.276 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-04-22T21:11:35.276 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-22T21:11:35.277 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-22T21:11:35.277 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-04-22T21:11:35.277 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-04-22T21:11:35.277 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-04-22T21:11:35.277 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-04-22T21:11:35.277 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-04-22T21:11:35.277 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-04-22T21:11:35.277 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-04-22T21:11:35.278 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-04-22T21:11:35.279 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-04-22T21:11:35.280 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-04-22T21:11:35.280 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-04-22T21:11:35.280 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-04-22T21:11:35.280 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-04-22T21:11:35.280 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-04-22T21:11:35.280 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-04-22T21:11:35.280 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-04-22T21:11:35.280 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-04-22T21:11:35.280 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-04-22T21:11:35.280 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-04-22T21:11:35.280 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-04-22T21:11:35.280 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-04-22T21:11:35.280 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-04-22T21:11:35.281 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-04-22T21:11:35.281 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-04-22T21:11:35.281 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-22T21:11:35.281 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-22T21:11:35.281 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-04-22T21:11:35.281 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-04-22T21:11:35.281 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-04-22T21:11:35.281 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-04-22T21:11:35.281 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-22T21:11:35.281 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-04-22T21:11:35.281 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-22T21:11:35.281 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-04-22T21:11:35.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-04-22T21:11:35.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-22T21:11:35.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-04-22T21:11:35.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-04-22T21:11:35.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-04-22T21:11:35.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-04-22T21:11:35.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-04-22T21:11:35.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-04-22T21:11:35.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-04-22T21:11:35.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-04-22T21:11:35.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-22T21:11:35.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-22T21:11:35.284 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-22T21:11:35.284 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-04-22T21:11:35.284 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-22T21:11:35.284 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-04-22T21:11:35.284 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-04-22T21:11:35.284 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-04-22T21:11:35.284 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-04-22T21:11:35.284 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-04-22T21:11:35.284 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-04-22T21:11:35.284 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-04-22T21:11:35.284 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-04-22T21:11:35.284 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-04-22T21:11:35.285 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-04-22T21:11:35.285 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-04-22T21:11:35.285 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-22T21:11:35.285 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-04-22T21:11:35.285 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-04-22T21:11:35.285 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-22T21:11:35.285 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-04-22T21:11:35.285 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-04-22T21:11:35.285 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-04-22T21:11:35.285 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-04-22T21:11:35.285 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-04-22T21:11:35.285 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-04-22T21:11:35.286 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-04-22T21:11:35.286 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-04-22T21:11:35.286 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-04-22T21:11:35.286 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-04-22T21:11:35.286 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-04-22T21:11:35.286 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-04-22T21:11:35.286 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-04-22T21:11:35.286 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-04-22T21:11:35.286 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-04-22T21:11:35.286 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-04-22T21:11:35.286 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-04-22T21:11:35.286 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-04-22T21:11:35.286 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-04-22T21:11:35.287 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-04-22T21:11:35.287 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-04-22T21:11:35.287 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-22T21:11:35.287 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-22T21:11:35.287 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-22T21:11:35.287 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-04-22T21:11:35.287 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-22T21:11:35.287 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-04-22T21:11:35.287 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-04-22T21:11:35.287 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-04-22T21:11:35.287 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-04-22T21:11:35.387 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:35 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-22T21:11:35.387 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:35 smithi130 ceph-mon[21867]: osdmap e61: 8 total, 8 up, 8 in 2024-04-22T21:11:35.387 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:35 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:35.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:35 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:11:35.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:35 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:11:35.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:35 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-22T21:11:35.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:35 smithi130 ceph-mon[26394]: osdmap e61: 8 total, 8 up, 8 in 2024-04-22T21:11:35.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:35 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:35.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:35 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:11:35.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:35 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:11:35.522 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-04-22T21:11:35.523 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-04-22T21:11:35.523 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-04-22T21:11:35.523 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-04-22T21:11:35.523 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-04-22T21:11:35.523 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-04-22T21:11:35.523 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-04-22T21:11:35.523 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-04-22T21:11:35.523 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-04-22T21:11:35.523 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-22T21:11:35.523 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-22T21:11:35.523 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-22T21:11:35.523 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-22T21:11:35.523 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-04-22T21:11:35.524 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-04-22T21:11:35.525 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-04-22T21:11:35.525 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-04-22T21:11:35.525 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-04-22T21:11:35.525 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-04-22T21:11:35.525 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-04-22T21:11:35.525 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-22T21:11:35.525 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-22T21:11:35.525 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-04-22T21:11:35.525 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-04-22T21:11:35.525 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-04-22T21:11:35.525 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-04-22T21:11:35.526 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-04-22T21:11:35.526 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-04-22T21:11:35.526 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-04-22T21:11:35.526 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-04-22T21:11:35.527 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-04-22T21:11:35.528 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-22T21:11:35.529 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-22T21:11:35.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-04-22T21:11:35.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-04-22T21:11:35.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-22T21:11:35.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-22T21:11:35.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-04-22T21:11:35.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-22T21:11:35.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-04-22T21:11:35.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-04-22T21:11:35.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-04-22T21:11:35.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-22T21:11:35.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-04-22T21:11:35.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-04-22T21:11:35.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-04-22T21:11:35.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-04-22T21:11:35.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-22T21:11:35.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-04-22T21:11:35.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-04-22T21:11:35.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-04-22T21:11:35.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-04-22T21:11:35.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-04-22T21:11:35.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-22T21:11:35.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-04-22T21:11:35.532 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-04-22T21:11:35.533 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-04-22T21:11:35.534 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-04-22T21:11:35.534 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-04-22T21:11:35.534 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-04-22T21:11:35.534 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-04-22T21:11:35.534 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-04-22T21:11:35.534 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-04-22T21:11:35.534 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-04-22T21:11:35.534 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-04-22T21:11:35.534 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-04-22T21:11:35.534 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-04-22T21:11:35.534 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-04-22T21:11:35.534 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-04-22T21:11:35.534 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-04-22T21:11:35.535 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-04-22T21:11:35.535 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-04-22T21:11:35.535 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-04-22T21:11:35.535 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-04-22T21:11:35.535 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-04-22T21:11:35.535 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-04-22T21:11:35.860 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-04-22T21:11:35.860 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-04-22T21:11:35.860 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-04-22T21:11:35.861 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-04-22T21:11:35.861 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-04-22T21:11:35.861 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-04-22T21:11:35.861 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-04-22T21:11:35.861 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-04-22T21:11:35.861 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=603.64605ms 2024-04-22T21:11:35.861 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-22T21:11:35.861 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-22T21:11:35.861 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-22T21:11:35.861 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-22T21:11:35.861 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-22T21:11:35.861 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-22T21:11:35.861 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-22T21:11:35.862 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-22T21:11:35.862 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-22T21:11:35.862 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-22T21:11:35.862 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:11:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:11:35+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-22T21:11:36.408 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:36 smithi130 ceph-mon[26394]: osdmap e62: 8 total, 8 up, 8 in 2024-04-22T21:11:36.408 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:36 smithi130 ceph-mon[26394]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-22T21:11:36.408 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:36 smithi130 ceph-mon[21867]: osdmap e62: 8 total, 8 up, 8 in 2024-04-22T21:11:36.408 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:36 smithi130 ceph-mon[21867]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-22T21:11:36.559 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:36 smithi150 ceph-mon[22676]: osdmap e62: 8 total, 8 up, 8 in 2024-04-22T21:11:36.559 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:36 smithi150 ceph-mon[22676]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-22T21:11:36.560 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:11:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:11:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:11:36.619 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:11:36.619 INFO:teuthology.orchestra.run.smithi130.stdout: "id": "a888925c-5a8e-4650-8e82-3763bd57d35d", 2024-04-22T21:11:36.619 INFO:teuthology.orchestra.run.smithi130.stdout: "name": "default", 2024-04-22T21:11:36.619 INFO:teuthology.orchestra.run.smithi130.stdout: "api_name": "default", 2024-04-22T21:11:36.619 INFO:teuthology.orchestra.run.smithi130.stdout: "is_master": "true", 2024-04-22T21:11:36.619 INFO:teuthology.orchestra.run.smithi130.stdout: "endpoints": [], 2024-04-22T21:11:36.619 INFO:teuthology.orchestra.run.smithi130.stdout: "hostnames": [], 2024-04-22T21:11:36.619 INFO:teuthology.orchestra.run.smithi130.stdout: "hostnames_s3website": [], 2024-04-22T21:11:36.620 INFO:teuthology.orchestra.run.smithi130.stdout: "master_zone": "", 2024-04-22T21:11:36.620 INFO:teuthology.orchestra.run.smithi130.stdout: "zones": [], 2024-04-22T21:11:36.620 INFO:teuthology.orchestra.run.smithi130.stdout: "placement_targets": [], 2024-04-22T21:11:36.620 INFO:teuthology.orchestra.run.smithi130.stdout: "default_placement": "", 2024-04-22T21:11:36.620 INFO:teuthology.orchestra.run.smithi130.stdout: "realm_id": "3d6051a8-76ed-49c6-9e41-0bd7c047ed53", 2024-04-22T21:11:36.620 INFO:teuthology.orchestra.run.smithi130.stdout: "sync_policy": { 2024-04-22T21:11:36.620 INFO:teuthology.orchestra.run.smithi130.stdout: "groups": [] 2024-04-22T21:11:36.620 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:11:36.620 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:11:37.172 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-04-22T21:11:39.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:39 smithi150 ceph-mon[22676]: pgmap v50: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-22T21:11:39.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:39 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:39.371 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:39 smithi130 ceph-mon[21867]: pgmap v50: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-22T21:11:39.371 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:39 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:39.371 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:39 smithi130 ceph-mon[26394]: pgmap v50: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-22T21:11:39.371 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:39 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:39.560 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:11:39.560 INFO:teuthology.orchestra.run.smithi130.stdout: "id": "d4e10ddf-884f-4986-8658-e51b3de892e3", 2024-04-22T21:11:39.560 INFO:teuthology.orchestra.run.smithi130.stdout: "name": "z", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "domain_root": "z.rgw.meta:root", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "control_pool": "z.rgw.control", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "gc_pool": "z.rgw.log:gc", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "lc_pool": "z.rgw.log:lc", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "log_pool": "z.rgw.log", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "roles_pool": "z.rgw.meta:roles", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-04-22T21:11:39.561 INFO:teuthology.orchestra.run.smithi130.stdout: "otp_pool": "z.rgw.otp", 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: "system_key": { 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: "access_key": "", 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: "secret_key": "" 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: "placement_pools": [ 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: "key": "default-placement", 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: "val": { 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: "index_pool": "z.rgw.buckets.index", 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: "storage_classes": { 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: "STANDARD": { 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: "data_pool": "z.rgw.buckets.data" 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:11:39.562 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:11:39.563 INFO:teuthology.orchestra.run.smithi130.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-04-22T21:11:39.563 INFO:teuthology.orchestra.run.smithi130.stdout: "index_type": 0 2024-04-22T21:11:39.563 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:11:39.563 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:11:39.563 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-04-22T21:11:39.563 INFO:teuthology.orchestra.run.smithi130.stdout: "realm_id": "3d6051a8-76ed-49c6-9e41-0bd7c047ed53", 2024-04-22T21:11:39.563 INFO:teuthology.orchestra.run.smithi130.stdout: "notif_pool": "z.rgw.log:notif" 2024-04-22T21:11:39.563 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:11:40.286 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:11:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:11:39] "GET /metrics HTTP/1.1" 200 191487 "" "Prometheus/2.33.4" 2024-04-22T21:11:40.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-04-22T21:11:41.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:41 smithi130 ceph-mon[26394]: pgmap v51: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-04-22T21:11:41.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:41 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:41.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:41 smithi130 ceph-mon[21867]: pgmap v51: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-04-22T21:11:41.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:41 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:41.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:41 smithi150 ceph-mon[22676]: pgmap v51: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-04-22T21:11:41.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:41 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:43.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:43 smithi150 ceph-mon[22676]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-04-22T21:11:43.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:43 smithi130 ceph-mon[26394]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-04-22T21:11:43.424 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:43 smithi130 ceph-mon[21867]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-04-22T21:11:44.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:44 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:44.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:44 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:44.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:44 smithi130 ceph-mon[21867]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T21:11:44.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:44 smithi130 ceph-mon[21867]: Reconfiguring daemon alertmanager.a on smithi130 2024-04-22T21:11:44.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:44 smithi130 ceph-mon[21867]: pgmap v53: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 732 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-04-22T21:11:44.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:44 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:44.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:44 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:44.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:44 smithi130 ceph-mon[26394]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T21:11:44.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:44 smithi130 ceph-mon[26394]: Reconfiguring daemon alertmanager.a on smithi130 2024-04-22T21:11:44.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:44 smithi130 ceph-mon[26394]: pgmap v53: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 732 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-04-22T21:11:44.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:44 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:44.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:44 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:44.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:44 smithi150 ceph-mon[22676]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T21:11:44.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:44 smithi150 ceph-mon[22676]: Reconfiguring daemon alertmanager.a on smithi130 2024-04-22T21:11:44.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:44 smithi150 ceph-mon[22676]: pgmap v53: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 732 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-04-22T21:11:45.660 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:45 smithi130 ceph-mon[21867]: osdmap e63: 8 total, 8 up, 8 in 2024-04-22T21:11:45.660 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:45 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2681278754' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T21:11:45.660 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:45 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T21:11:45.660 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:45 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:45.661 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:45 smithi130 ceph-mon[26394]: osdmap e63: 8 total, 8 up, 8 in 2024-04-22T21:11:45.661 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:45 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2681278754' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T21:11:45.661 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:45 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T21:11:45.661 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:45 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:45.860 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:45 smithi150 ceph-mon[22676]: osdmap e63: 8 total, 8 up, 8 in 2024-04-22T21:11:45.860 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:45 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2681278754' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T21:11:45.860 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:45 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T21:11:45.860 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:45 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:46.685 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:46 smithi130 systemd[1]: Stopping Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:11:46.685 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:46 smithi130 bash[46421]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-22T21:11:46.685 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:46 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-22T21:11:46.686 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:46 smithi130 ceph-mon[26394]: osdmap e64: 8 total, 8 up, 8 in 2024-04-22T21:11:46.686 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:46 smithi130 ceph-mon[26394]: pgmap v56: 65 pgs: 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 4 op/s 2024-04-22T21:11:46.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:46 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-22T21:11:46.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:46 smithi130 ceph-mon[21867]: osdmap e64: 8 total, 8 up, 8 in 2024-04-22T21:11:46.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:46 smithi130 ceph-mon[21867]: pgmap v56: 65 pgs: 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 4 op/s 2024-04-22T21:11:46.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:46 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-22T21:11:46.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:46 smithi150 ceph-mon[22676]: osdmap e64: 8 total, 8 up, 8 in 2024-04-22T21:11:46.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:46 smithi150 ceph-mon[22676]: pgmap v56: 65 pgs: 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 4 op/s 2024-04-22T21:11:46.805 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:11:46 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:11:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:11:46.993 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:46 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[44745]: level=info ts=2024-04-22T21:11:46.692Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-22T21:11:47.007 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:46 smithi130 podman[46434]: 2024-04-22 21:11:46.72324112 +0000 UTC m=+0.137032657 container died a2cc30afc0f2dcc2837498aff8aff4c19d9243c1109b35e763894f7f221670b3 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:47.306 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:47 smithi130 podman[46434]: 2024-04-22 21:11:47.083961727 +0000 UTC m=+0.497753263 container cleanup a2cc30afc0f2dcc2837498aff8aff4c19d9243c1109b35e763894f7f221670b3 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:47.306 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:47 smithi130 bash[46434]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a 2024-04-22T21:11:47.306 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:47 smithi130 bash[46463]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-22T21:11:47.632 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:47 smithi130 ceph-mon[26394]: osdmap e65: 8 total, 8 up, 8 in 2024-04-22T21:11:47.633 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:47 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2681278754' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T21:11:47.633 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:47 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T21:11:47.633 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:47 smithi130 ceph-mon[21867]: osdmap e65: 8 total, 8 up, 8 in 2024-04-22T21:11:47.633 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:47 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2681278754' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T21:11:47.633 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:47 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T21:11:47.633 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:47 smithi130 podman[46446]: 2024-04-22 21:11:47.334798659 +0000 UTC m=+0.609050884 container remove a2cc30afc0f2dcc2837498aff8aff4c19d9243c1109b35e763894f7f221670b3 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:47.634 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:47 smithi130 podman[46446]: 2024-04-22 21:11:47.410787657 +0000 UTC m=+0.685039882 volume remove 2bc002ca3ac776030b9d1a0e50b65b8a9f3290f680795bc27d46a0aa2001d7bb 2024-04-22T21:11:47.634 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:47 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-22T21:11:47.634 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:47 smithi130 systemd[1]: Stopped Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:11:47.634 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:47 smithi130 systemd[1]: Starting Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:11:47.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:47 smithi150 ceph-mon[22676]: osdmap e65: 8 total, 8 up, 8 in 2024-04-22T21:11:47.860 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:47 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2681278754' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T21:11:47.860 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:47 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T21:11:47.924 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:47 smithi130 podman[46533]: 2024-04-22 21:11:47.543514991 +0000 UTC m=+0.017961904 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-22T21:11:47.925 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:47 smithi130 podman[46533]: 2024-04-22 21:11:47.66241521 +0000 UTC m=+0.136862110 volume create e7b1bf86aa29acd52bcdab65b61cc65c92ec728cba337d813cf627eb4b1fa9ce 2024-04-22T21:11:47.925 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:47 smithi130 podman[46533]: 2024-04-22 21:11:47.762013848 +0000 UTC m=+0.236460748 container create 2176cd6f1d6a8bb4ce9befa302e267f9be2e4dbd2c7d4287b9be5383e8375f6c (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:48.306 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:48 smithi130 podman[46533]: 2024-04-22 21:11:48.039558603 +0000 UTC m=+0.514005522 container init 2176cd6f1d6a8bb4ce9befa302e267f9be2e4dbd2c7d4287b9be5383e8375f6c (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:48.306 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:48 smithi130 podman[46533]: 2024-04-22 21:11:48.04438803 +0000 UTC m=+0.518834939 container start 2176cd6f1d6a8bb4ce9befa302e267f9be2e4dbd2c7d4287b9be5383e8375f6c (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:48.306 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=info ts=2024-04-22T21:11:48.056Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-22T21:11:48.306 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=info ts=2024-04-22T21:11:48.056Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-22T21:11:48.306 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=info ts=2024-04-22T21:11:48.057Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.130 port=9094 2024-04-22T21:11:48.306 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=info ts=2024-04-22T21:11:48.059Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-22T21:11:48.307 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=info ts=2024-04-22T21:11:48.086Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T21:11:48.307 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=info ts=2024-04-22T21:11:48.087Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T21:11:48.307 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=info ts=2024-04-22T21:11:48.089Z caller=main.go:518 msg=Listening address=:9093 2024-04-22T21:11:48.307 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=info ts=2024-04-22T21:11:48.089Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-22T21:11:48.307 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:48 smithi130 bash[46533]: 2176cd6f1d6a8bb4ce9befa302e267f9be2e4dbd2c7d4287b9be5383e8375f6c 2024-04-22T21:11:48.307 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:48 smithi130 systemd[1]: Started Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:11:48.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:48 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-22T21:11:48.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:48 smithi150 ceph-mon[22676]: osdmap e66: 8 total, 8 up, 8 in 2024-04-22T21:11:48.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:48 smithi150 ceph-mon[22676]: pgmap v59: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-22T21:11:48.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:48 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:48.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-22T21:11:48.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-mon[26394]: osdmap e66: 8 total, 8 up, 8 in 2024-04-22T21:11:48.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-mon[26394]: pgmap v59: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-22T21:11:48.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:48.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-22T21:11:48.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-mon[21867]: osdmap e66: 8 total, 8 up, 8 in 2024-04-22T21:11:48.807 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-mon[21867]: pgmap v59: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-22T21:11:48.807 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:48 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:49.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:49 smithi130 ceph-mon[21867]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:11:49.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:49 smithi130 ceph-mon[21867]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:11:49.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:49 smithi130 ceph-mon[21867]: osdmap e67: 8 total, 8 up, 8 in 2024-04-22T21:11:49.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:49 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-22T21:11:49.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:49 smithi130 ceph-mon[26394]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:11:49.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:49 smithi130 ceph-mon[26394]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:11:49.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:49 smithi130 ceph-mon[26394]: osdmap e67: 8 total, 8 up, 8 in 2024-04-22T21:11:49.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:49 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-22T21:11:49.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:49 smithi150 ceph-mon[22676]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:11:49.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:49 smithi150 ceph-mon[22676]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:11:49.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:49 smithi150 ceph-mon[22676]: osdmap e67: 8 total, 8 up, 8 in 2024-04-22T21:11:49.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:49 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-22T21:11:50.306 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:11:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:11:49] "GET /metrics HTTP/1.1" 200 192909 "" "Prometheus/2.33.4" 2024-04-22T21:11:50.307 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:50 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=info ts=2024-04-22T21:11:50.060Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000801631s 2024-04-22T21:11:50.860 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:50 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-22T21:11:50.860 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:50 smithi150 ceph-mon[22676]: osdmap e68: 8 total, 8 up, 8 in 2024-04-22T21:11:50.860 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:50 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-22T21:11:50.860 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:50 smithi150 ceph-mon[22676]: pgmap v62: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:51.056 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:50 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-22T21:11:51.056 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:50 smithi130 ceph-mon[21867]: osdmap e68: 8 total, 8 up, 8 in 2024-04-22T21:11:51.056 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:50 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-22T21:11:51.056 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:50 smithi130 ceph-mon[21867]: pgmap v62: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:51.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:50 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-22T21:11:51.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:50 smithi130 ceph-mon[26394]: osdmap e68: 8 total, 8 up, 8 in 2024-04-22T21:11:51.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:50 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-22T21:11:51.057 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:50 smithi130 ceph-mon[26394]: pgmap v62: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:11:51.560 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 systemd[1]: Stopping Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:11:51.560 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 bash[39438]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T21:11:51.626 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:11:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "id": "206e74ab-aaac-425b-a863-fc650821f13a", 2024-04-22T21:11:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "epoch": 1, 2024-04-22T21:11:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "predecessor_uuid": "7ee2ba5a-2e63-4c5a-ac66-8ce17d8c5932", 2024-04-22T21:11:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "sync_status": [], 2024-04-22T21:11:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "period_map": { 2024-04-22T21:11:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "id": "206e74ab-aaac-425b-a863-fc650821f13a", 2024-04-22T21:11:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "zonegroups": [ 2024-04-22T21:11:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: "id": "a888925c-5a8e-4650-8e82-3763bd57d35d", 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: "name": "default", 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: "api_name": "default", 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: "is_master": "true", 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: "endpoints": [], 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: "hostnames": [], 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: "hostnames_s3website": [], 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: "master_zone": "d4e10ddf-884f-4986-8658-e51b3de892e3", 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: "zones": [ 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: "id": "d4e10ddf-884f-4986-8658-e51b3de892e3", 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: "name": "z", 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: "endpoints": [], 2024-04-22T21:11:51.627 INFO:teuthology.orchestra.run.smithi130.stdout: "log_meta": "false", 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: "log_data": "false", 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: "bucket_index_max_shards": 11, 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: "read_only": "false", 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: "tier_type": "", 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: "sync_from_all": "true", 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: "sync_from": [], 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: "redirect_zone": "" 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: "placement_targets": [ 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: "name": "default-placement", 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: "tags": [], 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: "storage_classes": [ 2024-04-22T21:11:51.628 INFO:teuthology.orchestra.run.smithi130.stdout: "STANDARD" 2024-04-22T21:11:51.629 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-04-22T21:11:51.629 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:11:51.629 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-04-22T21:11:51.629 INFO:teuthology.orchestra.run.smithi130.stdout: "default_placement": "default-placement", 2024-04-22T21:11:51.629 INFO:teuthology.orchestra.run.smithi130.stdout: "realm_id": "3d6051a8-76ed-49c6-9e41-0bd7c047ed53", 2024-04-22T21:11:51.629 INFO:teuthology.orchestra.run.smithi130.stdout: "sync_policy": { 2024-04-22T21:11:51.629 INFO:teuthology.orchestra.run.smithi130.stdout: "groups": [] 2024-04-22T21:11:51.629 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:11:51.629 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:11:51.629 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-04-22T21:11:51.630 INFO:teuthology.orchestra.run.smithi130.stdout: "short_zone_ids": [ 2024-04-22T21:11:51.630 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-04-22T21:11:51.630 INFO:teuthology.orchestra.run.smithi130.stdout: "key": "d4e10ddf-884f-4986-8658-e51b3de892e3", 2024-04-22T21:11:51.630 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 1486849519 2024-04-22T21:11:51.630 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:11:51.630 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-04-22T21:11:51.630 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:11:51.630 INFO:teuthology.orchestra.run.smithi130.stdout: "master_zonegroup": "a888925c-5a8e-4650-8e82-3763bd57d35d", 2024-04-22T21:11:51.630 INFO:teuthology.orchestra.run.smithi130.stdout: "master_zone": "d4e10ddf-884f-4986-8658-e51b3de892e3", 2024-04-22T21:11:51.630 INFO:teuthology.orchestra.run.smithi130.stdout: "period_config": { 2024-04-22T21:11:51.630 INFO:teuthology.orchestra.run.smithi130.stdout: "bucket_quota": { 2024-04-22T21:11:51.630 INFO:teuthology.orchestra.run.smithi130.stdout: "enabled": false, 2024-04-22T21:11:51.631 INFO:teuthology.orchestra.run.smithi130.stdout: "check_on_raw": false, 2024-04-22T21:11:51.631 INFO:teuthology.orchestra.run.smithi130.stdout: "max_size": -1, 2024-04-22T21:11:51.631 INFO:teuthology.orchestra.run.smithi130.stdout: "max_size_kb": 0, 2024-04-22T21:11:51.631 INFO:teuthology.orchestra.run.smithi130.stdout: "max_objects": -1 2024-04-22T21:11:51.631 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:11:51.631 INFO:teuthology.orchestra.run.smithi130.stdout: "user_quota": { 2024-04-22T21:11:51.631 INFO:teuthology.orchestra.run.smithi130.stdout: "enabled": false, 2024-04-22T21:11:51.631 INFO:teuthology.orchestra.run.smithi130.stdout: "check_on_raw": false, 2024-04-22T21:11:51.631 INFO:teuthology.orchestra.run.smithi130.stdout: "max_size": -1, 2024-04-22T21:11:51.631 INFO:teuthology.orchestra.run.smithi130.stdout: "max_size_kb": 0, 2024-04-22T21:11:51.631 INFO:teuthology.orchestra.run.smithi130.stdout: "max_objects": -1 2024-04-22T21:11:51.631 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:11:51.631 INFO:teuthology.orchestra.run.smithi130.stdout: "user_ratelimit": { 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: "max_read_ops": 0, 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: "max_write_ops": 0, 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: "max_read_bytes": 0, 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: "max_write_bytes": 0, 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: "enabled": false 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: "bucket_ratelimit": { 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: "max_read_ops": 0, 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: "max_write_ops": 0, 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: "max_read_bytes": 0, 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: "max_write_bytes": 0, 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: "enabled": false 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:11:51.632 INFO:teuthology.orchestra.run.smithi130.stdout: "anonymous_ratelimit": { 2024-04-22T21:11:51.633 INFO:teuthology.orchestra.run.smithi130.stdout: "max_read_ops": 0, 2024-04-22T21:11:51.633 INFO:teuthology.orchestra.run.smithi130.stdout: "max_write_ops": 0, 2024-04-22T21:11:51.633 INFO:teuthology.orchestra.run.smithi130.stdout: "max_read_bytes": 0, 2024-04-22T21:11:51.633 INFO:teuthology.orchestra.run.smithi130.stdout: "max_write_bytes": 0, 2024-04-22T21:11:51.633 INFO:teuthology.orchestra.run.smithi130.stdout: "enabled": false 2024-04-22T21:11:51.633 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:11:51.633 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:11:51.633 INFO:teuthology.orchestra.run.smithi130.stdout: "realm_id": "3d6051a8-76ed-49c6-9e41-0bd7c047ed53", 2024-04-22T21:11:51.633 INFO:teuthology.orchestra.run.smithi130.stdout: "realm_name": "r", 2024-04-22T21:11:51.633 INFO:teuthology.orchestra.run.smithi130.stdout: "realm_epoch": 2 2024-04-22T21:11:51.633 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:11:51.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-22T21:11:51.860 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-mon[22676]: osdmap e69: 8 total, 8 up, 8 in 2024-04-22T21:11:51.860 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-22T21:11:51.860 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:11:51.568Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T21:11:51.860 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:11:51.568Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T21:11:51.860 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:11:51.568Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T21:11:51.860 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:11:51.568Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T21:11:51.860 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:11:51.568Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T21:11:51.860 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:11:51.568Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T21:11:51.860 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:11:51.568Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T21:11:51.861 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:11:51.568Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T21:11:51.861 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:11:51.568Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T21:11:51.861 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:11:51.569Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T21:11:51.861 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:11:51.569Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T21:11:51.861 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[37774]: ts=2024-04-22T21:11:51.569Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T21:11:51.861 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:51 smithi150 podman[39450]: 2024-04-22 21:11:51.578417592 +0000 UTC m=+0.115478922 container died c106f1a0ac46d19a3392b0a5fa01459f9c4c5b4f6497f682cd978a27aa413285 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:51.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:51 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-22T21:11:51.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:51 smithi130 ceph-mon[26394]: osdmap e69: 8 total, 8 up, 8 in 2024-04-22T21:11:51.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:51 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-22T21:11:51.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:51 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-22T21:11:51.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:51 smithi130 ceph-mon[21867]: osdmap e69: 8 total, 8 up, 8 in 2024-04-22T21:11:51.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:51 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-22T21:11:52.290 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-04-22T21:11:52.294 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 podman[39450]: 2024-04-22 21:11:52.043300396 +0000 UTC m=+0.580361739 container cleanup c106f1a0ac46d19a3392b0a5fa01459f9c4c5b4f6497f682cd978a27aa413285 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:52.294 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 bash[39450]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T21:11:52.294 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 bash[39477]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T21:11:52.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 podman[39464]: 2024-04-22 21:11:52.302550785 +0000 UTC m=+0.726537085 container remove c106f1a0ac46d19a3392b0a5fa01459f9c4c5b4f6497f682cd978a27aa413285 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:52.546 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T21:11:52.546 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 systemd[1]: Stopped Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:11:52.546 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 systemd[1]: Starting Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:11:52.546 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 podman[39545]: 2024-04-22 21:11:52.437930038 +0000 UTC m=+0.019132807 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T21:11:52.546 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 podman[39545]: 2024-04-22 21:11:52.553588776 +0000 UTC m=+0.134791563 container create afc570412b78fd5f690739d7f29e7d0051980d22495614171d1e23fa2c47a0bc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:52.839 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-22T21:11:52.839 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-mon[22676]: osdmap e70: 8 total, 8 up, 8 in 2024-04-22T21:11:52.839 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-mon[22676]: pgmap v65: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-22T21:11:52.839 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 podman[39545]: 2024-04-22 21:11:52.76383639 +0000 UTC m=+0.345039135 container init afc570412b78fd5f690739d7f29e7d0051980d22495614171d1e23fa2c47a0bc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:52.839 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 podman[39545]: 2024-04-22 21:11:52.766455176 +0000 UTC m=+0.347657925 container start afc570412b78fd5f690739d7f29e7d0051980d22495614171d1e23fa2c47a0bc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:11:52.839 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.789Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T21:11:52.840 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.789Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T21:11:52.840 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.789Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T21:11:52.840 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.789Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi150 (none))" 2024-04-22T21:11:52.840 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.790Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T21:11:52.840 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.790Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T21:11:52.840 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.791Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T21:11:52.840 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.792Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T21:11:52.840 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.793Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T21:11:52.840 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.795Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T21:11:52.840 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.795Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.435µs 2024-04-22T21:11:52.840 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.795Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T21:11:52.840 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 bash[39545]: afc570412b78fd5f690739d7f29e7d0051980d22495614171d1e23fa2c47a0bc 2024-04-22T21:11:52.933 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:52 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-22T21:11:52.933 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:52 smithi130 ceph-mon[26394]: osdmap e70: 8 total, 8 up, 8 in 2024-04-22T21:11:52.934 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:52 smithi130 ceph-mon[26394]: pgmap v65: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-22T21:11:52.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:52 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2986294737' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-22T21:11:52.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:52 smithi130 ceph-mon[21867]: osdmap e70: 8 total, 8 up, 8 in 2024-04-22T21:11:52.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:52 smithi130 ceph-mon[21867]: pgmap v65: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-22T21:11:53.110 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 systemd[1]: Started Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:11:53.110 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.992Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-22T21:11:53.110 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.992Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-22T21:11:53.110 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.992Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.757µs wal_replay_duration=197.6921ms total_replay_duration=197.72916ms 2024-04-22T21:11:53.110 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.995Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T21:11:53.110 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.996Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T21:11:53.110 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:52.996Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T21:11:53.110 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:53.013Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.265798ms db_storage=825ns remote_storage=2.231µs web_handler=555ns query_engine=917ns scrape=554.131µs scrape_sd=50.431µs notify=37.499µs notify_sd=13.611µs rules=16.103331ms 2024-04-22T21:11:53.110 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:11:53.013Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T21:11:53.933 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:53.933 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:11:53.933 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:11:53.933 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.130:9093"}]: dispatch 2024-04-22T21:11:53.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.130:9093"}]: dispatch 2024-04-22T21:11:53.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:53.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:11:53.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:11:53.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.150:9095"}]: dispatch 2024-04-22T21:11:53.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.150:9095"}]: dispatch 2024-04-22T21:11:53.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:53.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:11:53.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:11:53.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.150:3000"}]: dispatch 2024-04-22T21:11:53.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.150:3000"}]: dispatch 2024-04-22T21:11:53.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:53.934 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:11:53.935 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:11:53.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:53.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:11:53.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:11:53.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.130:9093"}]: dispatch 2024-04-22T21:11:53.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.130:9093"}]: dispatch 2024-04-22T21:11:53.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:53.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:11:53.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:11:53.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.150:9095"}]: dispatch 2024-04-22T21:11:53.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.150:9095"}]: dispatch 2024-04-22T21:11:53.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:53.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:11:53.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:11:53.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.150:3000"}]: dispatch 2024-04-22T21:11:53.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.150:3000"}]: dispatch 2024-04-22T21:11:53.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:53.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:11:53.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:53 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:11:54.362 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled rgw.foo update... 2024-04-22T21:11:54.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:54.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:11:54.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:11:54.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.130:9093"}]: dispatch 2024-04-22T21:11:54.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.130:9093"}]: dispatch 2024-04-22T21:11:54.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:54.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:11:54.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:11:54.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.150:9095"}]: dispatch 2024-04-22T21:11:54.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.150:9095"}]: dispatch 2024-04-22T21:11:54.365 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:54.365 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:11:54.365 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:11:54.365 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.150:3000"}]: dispatch 2024-04-22T21:11:54.365 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.150:3000"}]: dispatch 2024-04-22T21:11:54.365 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:54.365 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:11:54.365 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:53 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:11:55.258 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:55 smithi150 ceph-mon[22676]: pgmap v66: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 187 B/s rd, 375 B/s wr, 1 op/s 2024-04-22T21:11:55.258 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:55 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:55.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:55 smithi130 ceph-mon[26394]: pgmap v66: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 187 B/s rd, 375 B/s wr, 1 op/s 2024-04-22T21:11:55.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:55 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:55.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:55 smithi130 ceph-mon[21867]: pgmap v66: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 187 B/s rd, 375 B/s wr, 1 op/s 2024-04-22T21:11:55.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:55 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:55.613 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph osd pool create foo' 2024-04-22T21:11:56.188 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:56 smithi130 ceph-mon[21867]: from='client.24589 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:11:56.188 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:56 smithi130 ceph-mon[21867]: Saving service rgw.foo spec with placement count:2 2024-04-22T21:11:56.189 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:56 smithi130 ceph-mon[26394]: from='client.24589 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:11:56.189 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:56 smithi130 ceph-mon[26394]: Saving service rgw.foo spec with placement count:2 2024-04-22T21:11:56.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:56 smithi150 ceph-mon[22676]: from='client.24589 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:11:56.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:56 smithi150 ceph-mon[22676]: Saving service rgw.foo spec with placement count:2 2024-04-22T21:11:57.283 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:57 smithi130 ceph-mon[21867]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-04-22T21:11:57.283 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:57 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:57.283 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:57 smithi130 ceph-mon[26394]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-04-22T21:11:57.283 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:57 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:57.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:57 smithi150 ceph-mon[22676]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-04-22T21:11:57.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:57 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:11:58.032 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'foo' created 2024-04-22T21:11:58.271 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:58 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1039924593' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-22T21:11:58.271 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:58 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-22T21:11:58.272 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:58 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1039924593' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-22T21:11:58.272 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:58 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-22T21:11:58.272 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:11:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=info ts=2024-04-22T21:11:58.062Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.00277709s 2024-04-22T21:11:58.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:58 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1039924593' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-22T21:11:58.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:58 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-22T21:11:58.919 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'rbd pool init foo' 2024-04-22T21:11:59.147 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:59 smithi130 ceph-mon[21867]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-22T21:11:59.147 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:59 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-22T21:11:59.147 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:11:59 smithi130 ceph-mon[21867]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T21:11:59.147 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:59 smithi130 ceph-mon[26394]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-22T21:11:59.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:59 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-22T21:11:59.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:11:59 smithi130 ceph-mon[26394]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T21:11:59.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:59 smithi150 ceph-mon[22676]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-22T21:11:59.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:59 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-22T21:11:59.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:11:59 smithi150 ceph-mon[22676]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T21:12:00.201 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:11:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:11:59] "GET /metrics HTTP/1.1" 200 196767 "" "Prometheus/2.33.4" 2024-04-22T21:12:00.201 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[21867]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T21:12:00.201 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.201 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.201 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.201 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.201 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi130.dfmola", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:12:00.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi130.dfmola", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T21:12:00.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.202 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:00.202 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[26394]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T21:12:00.202 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.202 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.202 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.202 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.202 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.202 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi130.dfmola", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:12:00.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi130.dfmola", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T21:12:00.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:00 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:00.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:00 smithi150 ceph-mon[22676]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T21:12:00.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:00 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:00 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:00 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:00 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:00 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:00 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi130.dfmola", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:12:00.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:00 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi130.dfmola", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T21:12:00.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:00 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:00.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:00 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:01.113 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:01 smithi130 ceph-mon[21867]: Saving service rgw.foo spec with placement count:2 2024-04-22T21:12:01.113 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:01 smithi130 ceph-mon[21867]: Deploying daemon rgw.foo.smithi130.dfmola on smithi130 2024-04-22T21:12:01.113 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:01 smithi130 ceph-mon[21867]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-04-22T21:12:01.113 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:01 smithi130 ceph-mon[21867]: osdmap e73: 8 total, 8 up, 8 in 2024-04-22T21:12:01.113 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:01 smithi130 ceph-mon[26394]: Saving service rgw.foo spec with placement count:2 2024-04-22T21:12:01.114 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:01 smithi130 ceph-mon[26394]: Deploying daemon rgw.foo.smithi130.dfmola on smithi130 2024-04-22T21:12:01.114 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:01 smithi130 ceph-mon[26394]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-04-22T21:12:01.114 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:01 smithi130 ceph-mon[26394]: osdmap e73: 8 total, 8 up, 8 in 2024-04-22T21:12:01.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:01 smithi150 ceph-mon[22676]: Saving service rgw.foo spec with placement count:2 2024-04-22T21:12:01.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:01 smithi150 ceph-mon[22676]: Deploying daemon rgw.foo.smithi130.dfmola on smithi130 2024-04-22T21:12:01.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:01 smithi150 ceph-mon[22676]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-04-22T21:12:01.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:01 smithi150 ceph-mon[22676]: osdmap e73: 8 total, 8 up, 8 in 2024-04-22T21:12:02.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:02 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1057793422' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-22T21:12:02.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:02 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1057793422' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-22T21:12:02.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:02 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1057793422' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-22T21:12:03.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:03 smithi130 ceph-mon[26394]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:12:03.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:03 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1057793422' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-22T21:12:03.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:03 smithi130 ceph-mon[26394]: osdmap e74: 8 total, 8 up, 8 in 2024-04-22T21:12:03.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:03 smithi130 ceph-mon[21867]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:12:03.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:03 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1057793422' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-22T21:12:03.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:03 smithi130 ceph-mon[21867]: osdmap e74: 8 total, 8 up, 8 in 2024-04-22T21:12:03.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:03 smithi150 ceph-mon[22676]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:12:03.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:03 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1057793422' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-22T21:12:03.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:03 smithi150 ceph-mon[22676]: osdmap e74: 8 total, 8 up, 8 in 2024-04-22T21:12:04.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:04 smithi150 ceph-mon[22676]: osdmap e75: 8 total, 8 up, 8 in 2024-04-22T21:12:04.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:04 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:04.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:04 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi150.xqvixd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:12:04.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:04 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi150.xqvixd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T21:12:04.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:04 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:04.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:04 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:04.430 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:04 smithi130 ceph-mon[21867]: osdmap e75: 8 total, 8 up, 8 in 2024-04-22T21:12:04.430 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:04 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:04.431 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:04 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi150.xqvixd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:12:04.431 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:04 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi150.xqvixd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T21:12:04.431 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:04 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:04.431 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:04 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:04.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:04 smithi130 ceph-mon[26394]: osdmap e75: 8 total, 8 up, 8 in 2024-04-22T21:12:04.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:04 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:04.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:04 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi150.xqvixd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:12:04.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:04 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi150.xqvixd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T21:12:04.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:04 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:04.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:04 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:04.899 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch apply iscsi foo u p' 2024-04-22T21:12:05.206 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:05 smithi130 ceph-mon[21867]: Deploying daemon rgw.foo.smithi150.xqvixd on smithi150 2024-04-22T21:12:05.206 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:05 smithi130 ceph-mon[21867]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:12:05.207 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:05 smithi130 ceph-mon[21867]: osdmap e76: 8 total, 8 up, 8 in 2024-04-22T21:12:05.207 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:05 smithi130 ceph-mon[26394]: Deploying daemon rgw.foo.smithi150.xqvixd on smithi150 2024-04-22T21:12:05.207 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:05 smithi130 ceph-mon[26394]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:12:05.207 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:05 smithi130 ceph-mon[26394]: osdmap e76: 8 total, 8 up, 8 in 2024-04-22T21:12:05.341 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:05 smithi150 ceph-mon[22676]: Deploying daemon rgw.foo.smithi150.xqvixd on smithi150 2024-04-22T21:12:05.341 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:05 smithi150 ceph-mon[22676]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:12:05.341 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:05 smithi150 ceph-mon[22676]: osdmap e76: 8 total, 8 up, 8 in 2024-04-22T21:12:06.370 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:12:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:12:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:12:06.928 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled iscsi.foo update... 2024-04-22T21:12:07.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:07 smithi130 ceph-mon[21867]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s rd, 5.9 KiB/s wr, 188 op/s 2024-04-22T21:12:07.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:07 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:07.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:07 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:07.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:07 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:12:07.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:07 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:07.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:07 smithi130 ceph-mon[26394]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s rd, 5.9 KiB/s wr, 188 op/s 2024-04-22T21:12:07.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:07 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:07.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:07 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:07.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:07 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:12:07.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:07 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:07.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:07 smithi150 ceph-mon[22676]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s rd, 5.9 KiB/s wr, 188 op/s 2024-04-22T21:12:07.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:07 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:07.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:07 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:07.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:07 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:12:07.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:07 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:07.898 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'sleep 180' 2024-04-22T21:12:08.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:08 smithi130 ceph-mon[21867]: from='client.24646 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:12:08.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:08 smithi130 ceph-mon[21867]: Saving service iscsi.foo spec with placement count:1 2024-04-22T21:12:08.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:08 smithi130 ceph-mon[26394]: from='client.24646 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:12:08.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:08 smithi130 ceph-mon[26394]: Saving service iscsi.foo spec with placement count:1 2024-04-22T21:12:08.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:08 smithi150 ceph-mon[22676]: from='client.24646 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:12:08.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:08 smithi150 ceph-mon[22676]: Saving service iscsi.foo spec with placement count:1 2024-04-22T21:12:09.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:09 smithi150 ceph-mon[22676]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 5.8 KiB/s wr, 187 op/s 2024-04-22T21:12:09.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:09 smithi150 ceph-mon[22676]: osdmap e77: 8 total, 8 up, 8 in 2024-04-22T21:12:09.449 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:09 smithi130 ceph-mon[21867]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 5.8 KiB/s wr, 187 op/s 2024-04-22T21:12:09.449 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:09 smithi130 ceph-mon[21867]: osdmap e77: 8 total, 8 up, 8 in 2024-04-22T21:12:09.450 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:09 smithi130 ceph-mon[26394]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 5.8 KiB/s wr, 187 op/s 2024-04-22T21:12:09.450 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:09 smithi130 ceph-mon[26394]: osdmap e77: 8 total, 8 up, 8 in 2024-04-22T21:12:10.055 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:12:09 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:12:09] "GET /metrics HTTP/1.1" 200 196767 "" "Prometheus/2.33.4" 2024-04-22T21:12:10.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]: dispatch 2024-04-22T21:12:10.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-04-22T21:12:10.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-04-22T21:12:10.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-22T21:12:10.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:12:10.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:12:10.363 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]: dispatch 2024-04-22T21:12:10.363 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-04-22T21:12:10.363 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-04-22T21:12:10.363 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-22T21:12:10.363 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:12:10.363 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:12:10.363 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]: dispatch 2024-04-22T21:12:10.363 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-04-22T21:12:10.363 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-04-22T21:12:10.363 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-22T21:12:10.363 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:12:10.363 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:12:11.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[26394]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 5.0 KiB/s wr, 161 op/s 2024-04-22T21:12:11.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]': finished 2024-04-22T21:12:11.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-04-22T21:12:11.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-04-22T21:12:11.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-04-22T21:12:11.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[26394]: osdmap e78: 8 total, 8 up, 8 in 2024-04-22T21:12:11.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:11.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:11.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[21867]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 5.0 KiB/s wr, 161 op/s 2024-04-22T21:12:11.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]': finished 2024-04-22T21:12:11.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-04-22T21:12:11.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-04-22T21:12:11.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-04-22T21:12:11.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[21867]: osdmap e78: 8 total, 8 up, 8 in 2024-04-22T21:12:11.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:11.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:11.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:11 smithi150 ceph-mon[22676]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 5.0 KiB/s wr, 161 op/s 2024-04-22T21:12:11.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]': finished 2024-04-22T21:12:11.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-04-22T21:12:11.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-04-22T21:12:11.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-04-22T21:12:11.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:11 smithi150 ceph-mon[22676]: osdmap e78: 8 total, 8 up, 8 in 2024-04-22T21:12:11.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:11.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:12.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:12 smithi150 ceph-mon[22676]: osdmap e79: 8 total, 8 up, 8 in 2024-04-22T21:12:12.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:12 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:12.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:12 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:12.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:12 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:12.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:12 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:12.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:12 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:12:12.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:12 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-22T21:12:12.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:12 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:12.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[26394]: osdmap e79: 8 total, 8 up, 8 in 2024-04-22T21:12:12.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:12.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:12.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:12.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:12.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:12:12.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-22T21:12:12.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:12.387 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[21867]: osdmap e79: 8 total, 8 up, 8 in 2024-04-22T21:12:12.387 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:12.387 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:12.387 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:12.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:12.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:12:12.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-22T21:12:12.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:12 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:13.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:13 smithi150 ceph-mon[22676]: Checking dashboard <-> RGW credentials 2024-04-22T21:12:13.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:13 smithi150 ceph-mon[22676]: Checking pool "foo" exists for service iscsi.foo 2024-04-22T21:12:13.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:13 smithi150 ceph-mon[22676]: Deploying daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:12:13.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:13 smithi150 ceph-mon[22676]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 108 KiB/s rd, 170 B/s wr, 166 op/s 2024-04-22T21:12:13.462 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:13 smithi130 ceph-mon[21867]: Checking dashboard <-> RGW credentials 2024-04-22T21:12:13.463 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:13 smithi130 ceph-mon[21867]: Checking pool "foo" exists for service iscsi.foo 2024-04-22T21:12:13.463 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:13 smithi130 ceph-mon[21867]: Deploying daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:12:13.463 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:13 smithi130 ceph-mon[21867]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 108 KiB/s rd, 170 B/s wr, 166 op/s 2024-04-22T21:12:13.463 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:13 smithi130 ceph-mon[26394]: Checking dashboard <-> RGW credentials 2024-04-22T21:12:13.463 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:13 smithi130 ceph-mon[26394]: Checking pool "foo" exists for service iscsi.foo 2024-04-22T21:12:13.463 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:13 smithi130 ceph-mon[26394]: Deploying daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:12:13.463 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:13 smithi130 ceph-mon[26394]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 108 KiB/s rd, 170 B/s wr, 166 op/s 2024-04-22T21:12:15.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:15 smithi150 ceph-mon[22676]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 108 KiB/s rd, 170 B/s wr, 166 op/s 2024-04-22T21:12:15.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:15 smithi150 ceph-mon[22676]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-22T21:12:15.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:15 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:15.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:15 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:15.360 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:15 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:12:15.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:15 smithi130 ceph-mon[26394]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 108 KiB/s rd, 170 B/s wr, 166 op/s 2024-04-22T21:12:15.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:15 smithi130 ceph-mon[26394]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-22T21:12:15.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:15 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:15.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:15 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:15.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:15 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:12:15.431 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:15 smithi130 ceph-mon[21867]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 108 KiB/s rd, 170 B/s wr, 166 op/s 2024-04-22T21:12:15.431 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:15 smithi130 ceph-mon[21867]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-22T21:12:15.431 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:15 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:15.431 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:15 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:15.431 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:15 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:12:16.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:16 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:16.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:16 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4195607476' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:12:16.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:16 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1507996790' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4195343556"}]: dispatch 2024-04-22T21:12:16.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:16 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4195343556"}]: dispatch 2024-04-22T21:12:16.395 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:16 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:16.395 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:16 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4195607476' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:12:16.395 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:16 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1507996790' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4195343556"}]: dispatch 2024-04-22T21:12:16.396 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:16 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4195343556"}]: dispatch 2024-04-22T21:12:16.396 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:16 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:16.396 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:16 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4195607476' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:12:16.396 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:16 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1507996790' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4195343556"}]: dispatch 2024-04-22T21:12:16.396 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:16 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4195343556"}]: dispatch 2024-04-22T21:12:16.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:12:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:12:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:12:17.188 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:17 smithi150 ceph-mon[22676]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 583 B/s wr, 150 op/s; 6 B/s, 0 objects/s recovering 2024-04-22T21:12:17.188 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:17 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4195343556"}]': finished 2024-04-22T21:12:17.188 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:17 smithi150 ceph-mon[22676]: osdmap e80: 8 total, 8 up, 8 in 2024-04-22T21:12:17.188 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:17 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/948623315' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1551796354"}]: dispatch 2024-04-22T21:12:17.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:17 smithi130 ceph-mon[26394]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 583 B/s wr, 150 op/s; 6 B/s, 0 objects/s recovering 2024-04-22T21:12:17.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:17 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4195343556"}]': finished 2024-04-22T21:12:17.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:17 smithi130 ceph-mon[26394]: osdmap e80: 8 total, 8 up, 8 in 2024-04-22T21:12:17.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:17 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/948623315' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1551796354"}]: dispatch 2024-04-22T21:12:17.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:17 smithi130 ceph-mon[21867]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 583 B/s wr, 150 op/s; 6 B/s, 0 objects/s recovering 2024-04-22T21:12:17.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:17 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4195343556"}]': finished 2024-04-22T21:12:17.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:17 smithi130 ceph-mon[21867]: osdmap e80: 8 total, 8 up, 8 in 2024-04-22T21:12:17.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:17 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/948623315' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1551796354"}]: dispatch 2024-04-22T21:12:18.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:18 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/948623315' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1551796354"}]': finished 2024-04-22T21:12:18.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:18 smithi130 ceph-mon[26394]: osdmap e81: 8 total, 8 up, 8 in 2024-04-22T21:12:18.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:18 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4056067423' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3442044605"}]: dispatch 2024-04-22T21:12:18.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:18 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/948623315' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1551796354"}]': finished 2024-04-22T21:12:18.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:18 smithi130 ceph-mon[21867]: osdmap e81: 8 total, 8 up, 8 in 2024-04-22T21:12:18.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:18 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4056067423' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3442044605"}]: dispatch 2024-04-22T21:12:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:18 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/948623315' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1551796354"}]': finished 2024-04-22T21:12:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:18 smithi150 ceph-mon[22676]: osdmap e81: 8 total, 8 up, 8 in 2024-04-22T21:12:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:18 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4056067423' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3442044605"}]: dispatch 2024-04-22T21:12:19.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 443 B/s wr, 8 op/s; 6 B/s, 0 objects/s recovering 2024-04-22T21:12:19.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4056067423' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3442044605"}]': finished 2024-04-22T21:12:19.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: osdmap e82: 8 total, 8 up, 8 in 2024-04-22T21:12:19.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:12:19.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:12:19.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi130"}]: dispatch 2024-04-22T21:12:19.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:19.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:12:19.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2004103732' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/4050722828"}]: dispatch 2024-04-22T21:12:19.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/4050722828"}]: dispatch 2024-04-22T21:12:19.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 443 B/s wr, 8 op/s; 6 B/s, 0 objects/s recovering 2024-04-22T21:12:19.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4056067423' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3442044605"}]': finished 2024-04-22T21:12:19.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: osdmap e82: 8 total, 8 up, 8 in 2024-04-22T21:12:19.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:12:19.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:12:19.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi130"}]: dispatch 2024-04-22T21:12:19.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:19.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:12:19.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2004103732' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/4050722828"}]: dispatch 2024-04-22T21:12:19.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/4050722828"}]: dispatch 2024-04-22T21:12:19.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 443 B/s wr, 8 op/s; 6 B/s, 0 objects/s recovering 2024-04-22T21:12:19.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4056067423' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3442044605"}]': finished 2024-04-22T21:12:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: osdmap e82: 8 total, 8 up, 8 in 2024-04-22T21:12:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:12:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:12:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi130"}]: dispatch 2024-04-22T21:12:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:12:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:12:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2004103732' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/4050722828"}]: dispatch 2024-04-22T21:12:19.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/4050722828"}]: dispatch 2024-04-22T21:12:19.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:12:20.305 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:12:19 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:12:19] "GET /metrics HTTP/1.1" 200 207117 "" "Prometheus/2.33.4" 2024-04-22T21:12:20.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:12:20.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[21867]: Adding iSCSI gateway http://:@172.21.15.130:5000 to Dashboard 2024-04-22T21:12:20.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:12:20.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi130"}]: dispatch 2024-04-22T21:12:20.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[21867]: Checking dashboard <-> RGW credentials 2024-04-22T21:12:20.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/4050722828"}]': finished 2024-04-22T21:12:20.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[21867]: osdmap e83: 8 total, 8 up, 8 in 2024-04-22T21:12:20.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3541617209' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1754652716"}]: dispatch 2024-04-22T21:12:20.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1754652716"}]: dispatch 2024-04-22T21:12:20.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[26394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:12:20.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[26394]: Adding iSCSI gateway http://:@172.21.15.130:5000 to Dashboard 2024-04-22T21:12:20.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[26394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:12:20.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[26394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi130"}]: dispatch 2024-04-22T21:12:20.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[26394]: Checking dashboard <-> RGW credentials 2024-04-22T21:12:20.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/4050722828"}]': finished 2024-04-22T21:12:20.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[26394]: osdmap e83: 8 total, 8 up, 8 in 2024-04-22T21:12:20.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3541617209' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1754652716"}]: dispatch 2024-04-22T21:12:20.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:20 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1754652716"}]: dispatch 2024-04-22T21:12:20.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:20 smithi150 ceph-mon[22676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:12:20.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:20 smithi150 ceph-mon[22676]: Adding iSCSI gateway http://:@172.21.15.130:5000 to Dashboard 2024-04-22T21:12:20.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:20 smithi150 ceph-mon[22676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:12:20.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:20 smithi150 ceph-mon[22676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi130"}]: dispatch 2024-04-22T21:12:20.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:20 smithi150 ceph-mon[22676]: Checking dashboard <-> RGW credentials 2024-04-22T21:12:20.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:20 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/4050722828"}]': finished 2024-04-22T21:12:20.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:20 smithi150 ceph-mon[22676]: osdmap e83: 8 total, 8 up, 8 in 2024-04-22T21:12:20.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:20 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3541617209' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1754652716"}]: dispatch 2024-04-22T21:12:20.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:20 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1754652716"}]: dispatch 2024-04-22T21:12:21.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:21 smithi130 ceph-mon[21867]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:12:21.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:21 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1754652716"}]': finished 2024-04-22T21:12:21.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:21 smithi130 ceph-mon[21867]: osdmap e84: 8 total, 8 up, 8 in 2024-04-22T21:12:21.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:21 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3421594322' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1814259082"}]: dispatch 2024-04-22T21:12:21.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:21 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1814259082"}]: dispatch 2024-04-22T21:12:21.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:21 smithi130 ceph-mon[26394]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:12:21.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:21 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1754652716"}]': finished 2024-04-22T21:12:21.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:21 smithi130 ceph-mon[26394]: osdmap e84: 8 total, 8 up, 8 in 2024-04-22T21:12:21.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:21 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3421594322' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1814259082"}]: dispatch 2024-04-22T21:12:21.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:21 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1814259082"}]: dispatch 2024-04-22T21:12:21.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:21 smithi150 ceph-mon[22676]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:12:21.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:21 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1754652716"}]': finished 2024-04-22T21:12:21.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:21 smithi150 ceph-mon[22676]: osdmap e84: 8 total, 8 up, 8 in 2024-04-22T21:12:21.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:21 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3421594322' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1814259082"}]: dispatch 2024-04-22T21:12:21.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:21 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1814259082"}]: dispatch 2024-04-22T21:12:22.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:22 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1814259082"}]': finished 2024-04-22T21:12:22.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:22 smithi150 ceph-mon[22676]: osdmap e85: 8 total, 8 up, 8 in 2024-04-22T21:12:22.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:22 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1208178870' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/4050722828"}]: dispatch 2024-04-22T21:12:22.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:22 smithi150 ceph-mon[22676]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T21:12:22.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:22 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1814259082"}]': finished 2024-04-22T21:12:22.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:22 smithi130 ceph-mon[21867]: osdmap e85: 8 total, 8 up, 8 in 2024-04-22T21:12:22.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:22 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1208178870' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/4050722828"}]: dispatch 2024-04-22T21:12:22.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:22 smithi130 ceph-mon[21867]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T21:12:22.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:22 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1814259082"}]': finished 2024-04-22T21:12:22.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:22 smithi130 ceph-mon[26394]: osdmap e85: 8 total, 8 up, 8 in 2024-04-22T21:12:22.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:22 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1208178870' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/4050722828"}]: dispatch 2024-04-22T21:12:22.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:22 smithi130 ceph-mon[26394]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T21:12:23.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:23 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1208178870' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/4050722828"}]': finished 2024-04-22T21:12:23.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:23 smithi150 ceph-mon[22676]: osdmap e86: 8 total, 8 up, 8 in 2024-04-22T21:12:23.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:23 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1752930445' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2290497716"}]: dispatch 2024-04-22T21:12:23.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:23 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1208178870' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/4050722828"}]': finished 2024-04-22T21:12:23.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:23 smithi130 ceph-mon[26394]: osdmap e86: 8 total, 8 up, 8 in 2024-04-22T21:12:23.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:23 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1752930445' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2290497716"}]: dispatch 2024-04-22T21:12:23.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:23 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1208178870' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/4050722828"}]': finished 2024-04-22T21:12:23.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:23 smithi130 ceph-mon[21867]: osdmap e86: 8 total, 8 up, 8 in 2024-04-22T21:12:23.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:23 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1752930445' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2290497716"}]: dispatch 2024-04-22T21:12:24.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:24 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1752930445' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2290497716"}]': finished 2024-04-22T21:12:24.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:24 smithi150 ceph-mon[22676]: osdmap e87: 8 total, 8 up, 8 in 2024-04-22T21:12:24.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:24 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2043172732' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/636209570"}]: dispatch 2024-04-22T21:12:24.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:24 smithi150 ceph-mon[22676]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T21:12:24.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:24 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1752930445' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2290497716"}]': finished 2024-04-22T21:12:24.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:24 smithi130 ceph-mon[26394]: osdmap e87: 8 total, 8 up, 8 in 2024-04-22T21:12:24.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:24 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2043172732' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/636209570"}]: dispatch 2024-04-22T21:12:24.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:24 smithi130 ceph-mon[26394]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T21:12:24.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:24 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1752930445' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2290497716"}]': finished 2024-04-22T21:12:24.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:24 smithi130 ceph-mon[21867]: osdmap e87: 8 total, 8 up, 8 in 2024-04-22T21:12:24.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:24 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2043172732' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/636209570"}]: dispatch 2024-04-22T21:12:24.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:24 smithi130 ceph-mon[21867]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T21:12:25.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:25 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2043172732' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/636209570"}]': finished 2024-04-22T21:12:25.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:25 smithi150 ceph-mon[22676]: osdmap e88: 8 total, 8 up, 8 in 2024-04-22T21:12:25.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:25 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3277926008' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2397194687"}]: dispatch 2024-04-22T21:12:25.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:25 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2397194687"}]: dispatch 2024-04-22T21:12:25.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:25 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:12:25.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:25 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2043172732' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/636209570"}]': finished 2024-04-22T21:12:25.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:25 smithi130 ceph-mon[26394]: osdmap e88: 8 total, 8 up, 8 in 2024-04-22T21:12:25.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:25 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3277926008' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2397194687"}]: dispatch 2024-04-22T21:12:25.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:25 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2397194687"}]: dispatch 2024-04-22T21:12:25.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:25 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:12:25.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:25 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2043172732' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/636209570"}]': finished 2024-04-22T21:12:25.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:25 smithi130 ceph-mon[21867]: osdmap e88: 8 total, 8 up, 8 in 2024-04-22T21:12:25.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:25 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3277926008' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2397194687"}]: dispatch 2024-04-22T21:12:25.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:25 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2397194687"}]: dispatch 2024-04-22T21:12:25.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:25 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:12:26.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:12:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:12:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:12:26.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:26 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2397194687"}]': finished 2024-04-22T21:12:26.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:26 smithi150 ceph-mon[22676]: osdmap e89: 8 total, 8 up, 8 in 2024-04-22T21:12:26.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:26 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3631385429' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1814259082"}]: dispatch 2024-04-22T21:12:26.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:26 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1814259082"}]: dispatch 2024-04-22T21:12:26.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:26 smithi150 ceph-mon[22676]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:26.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:26 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2397194687"}]': finished 2024-04-22T21:12:26.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:26 smithi130 ceph-mon[26394]: osdmap e89: 8 total, 8 up, 8 in 2024-04-22T21:12:26.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:26 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3631385429' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1814259082"}]: dispatch 2024-04-22T21:12:26.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:26 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1814259082"}]: dispatch 2024-04-22T21:12:26.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:26 smithi130 ceph-mon[26394]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:26.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:26 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2397194687"}]': finished 2024-04-22T21:12:26.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:26 smithi130 ceph-mon[21867]: osdmap e89: 8 total, 8 up, 8 in 2024-04-22T21:12:26.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:26 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3631385429' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1814259082"}]: dispatch 2024-04-22T21:12:26.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:26 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1814259082"}]: dispatch 2024-04-22T21:12:26.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:26 smithi130 ceph-mon[21867]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:27.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:27 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1814259082"}]': finished 2024-04-22T21:12:27.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:27 smithi150 ceph-mon[22676]: osdmap e90: 8 total, 8 up, 8 in 2024-04-22T21:12:27.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:27 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3558098559' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2845297619"}]: dispatch 2024-04-22T21:12:27.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:27 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1814259082"}]': finished 2024-04-22T21:12:27.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:27 smithi130 ceph-mon[21867]: osdmap e90: 8 total, 8 up, 8 in 2024-04-22T21:12:27.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:27 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3558098559' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2845297619"}]: dispatch 2024-04-22T21:12:27.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:27 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/1814259082"}]': finished 2024-04-22T21:12:27.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:27 smithi130 ceph-mon[26394]: osdmap e90: 8 total, 8 up, 8 in 2024-04-22T21:12:27.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:27 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3558098559' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2845297619"}]: dispatch 2024-04-22T21:12:28.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:28 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3558098559' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2845297619"}]': finished 2024-04-22T21:12:28.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:28 smithi130 ceph-mon[21867]: osdmap e91: 8 total, 8 up, 8 in 2024-04-22T21:12:28.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:28 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1429389778' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4252030715"}]: dispatch 2024-04-22T21:12:28.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:28 smithi130 ceph-mon[21867]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:28.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:28 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3558098559' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2845297619"}]': finished 2024-04-22T21:12:28.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:28 smithi130 ceph-mon[26394]: osdmap e91: 8 total, 8 up, 8 in 2024-04-22T21:12:28.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:28 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1429389778' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4252030715"}]: dispatch 2024-04-22T21:12:28.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:28 smithi130 ceph-mon[26394]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:28.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:28 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3558098559' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2845297619"}]': finished 2024-04-22T21:12:28.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:28 smithi150 ceph-mon[22676]: osdmap e91: 8 total, 8 up, 8 in 2024-04-22T21:12:28.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:28 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1429389778' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4252030715"}]: dispatch 2024-04-22T21:12:28.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:28 smithi150 ceph-mon[22676]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:29.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:29 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1429389778' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4252030715"}]': finished 2024-04-22T21:12:29.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:29 smithi130 ceph-mon[21867]: osdmap e92: 8 total, 8 up, 8 in 2024-04-22T21:12:29.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:29 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1947520152' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3720321995"}]: dispatch 2024-04-22T21:12:29.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:29 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1429389778' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4252030715"}]': finished 2024-04-22T21:12:29.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:29 smithi130 ceph-mon[26394]: osdmap e92: 8 total, 8 up, 8 in 2024-04-22T21:12:29.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:29 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1947520152' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3720321995"}]: dispatch 2024-04-22T21:12:29.860 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:29 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1429389778' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4252030715"}]': finished 2024-04-22T21:12:29.861 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:29 smithi150 ceph-mon[22676]: osdmap e92: 8 total, 8 up, 8 in 2024-04-22T21:12:29.861 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:29 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1947520152' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3720321995"}]: dispatch 2024-04-22T21:12:30.305 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:12:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:12:29] "GET /metrics HTTP/1.1" 200 208009 "" "Prometheus/2.33.4" 2024-04-22T21:12:30.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:30 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1947520152' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3720321995"}]': finished 2024-04-22T21:12:30.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:30 smithi130 ceph-mon[26394]: osdmap e93: 8 total, 8 up, 8 in 2024-04-22T21:12:30.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:30 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3022595438' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1754652716"}]: dispatch 2024-04-22T21:12:30.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:30 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1754652716"}]: dispatch 2024-04-22T21:12:30.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:30 smithi130 ceph-mon[26394]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:12:30.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:30 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1947520152' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3720321995"}]': finished 2024-04-22T21:12:30.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:30 smithi130 ceph-mon[21867]: osdmap e93: 8 total, 8 up, 8 in 2024-04-22T21:12:30.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:30 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3022595438' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1754652716"}]: dispatch 2024-04-22T21:12:30.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:30 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1754652716"}]: dispatch 2024-04-22T21:12:30.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:30 smithi130 ceph-mon[21867]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:12:30.861 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:30 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1947520152' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3720321995"}]': finished 2024-04-22T21:12:30.861 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:30 smithi150 ceph-mon[22676]: osdmap e93: 8 total, 8 up, 8 in 2024-04-22T21:12:30.861 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:30 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3022595438' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1754652716"}]: dispatch 2024-04-22T21:12:30.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:30 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1754652716"}]: dispatch 2024-04-22T21:12:30.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:30 smithi150 ceph-mon[22676]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:12:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:31 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1754652716"}]': finished 2024-04-22T21:12:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:31 smithi130 ceph-mon[26394]: osdmap e94: 8 total, 8 up, 8 in 2024-04-22T21:12:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:31 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4135740408' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4177177338"}]: dispatch 2024-04-22T21:12:31.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:31 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1754652716"}]': finished 2024-04-22T21:12:31.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:31 smithi130 ceph-mon[21867]: osdmap e94: 8 total, 8 up, 8 in 2024-04-22T21:12:31.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:31 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4135740408' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4177177338"}]: dispatch 2024-04-22T21:12:31.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:31 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/1754652716"}]': finished 2024-04-22T21:12:31.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:31 smithi150 ceph-mon[22676]: osdmap e94: 8 total, 8 up, 8 in 2024-04-22T21:12:31.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:31 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4135740408' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4177177338"}]: dispatch 2024-04-22T21:12:32.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:32 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4135740408' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4177177338"}]': finished 2024-04-22T21:12:32.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:32 smithi130 ceph-mon[21867]: osdmap e95: 8 total, 8 up, 8 in 2024-04-22T21:12:32.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:32 smithi130 ceph-mon[21867]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:32.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:32 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4135740408' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4177177338"}]': finished 2024-04-22T21:12:32.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:32 smithi130 ceph-mon[26394]: osdmap e95: 8 total, 8 up, 8 in 2024-04-22T21:12:32.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:32 smithi130 ceph-mon[26394]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:32.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:32 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4135740408' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/4177177338"}]': finished 2024-04-22T21:12:32.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:32 smithi150 ceph-mon[22676]: osdmap e95: 8 total, 8 up, 8 in 2024-04-22T21:12:32.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:32 smithi150 ceph-mon[22676]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:35.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:35 smithi130 ceph-mon[21867]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-04-22T21:12:35.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:35 smithi130 ceph-mon[26394]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-04-22T21:12:35.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:35 smithi150 ceph-mon[22676]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-04-22T21:12:36.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:36 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:12:36.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:36 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:12:36.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:36 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:12:36.859 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:12:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:12:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:12:37.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:37 smithi130 ceph-mon[26394]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-22T21:12:37.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:37 smithi130 ceph-mon[21867]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-22T21:12:37.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:37 smithi150 ceph-mon[22676]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-22T21:12:39.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:39 smithi150 ceph-mon[22676]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:39 smithi130 ceph-mon[21867]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:39.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:39 smithi130 ceph-mon[26394]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:40.305 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:12:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:12:39] "GET /metrics HTTP/1.1" 200 208009 "" "Prometheus/2.33.4" 2024-04-22T21:12:41.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:41 smithi150 ceph-mon[22676]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T21:12:41.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:41 smithi130 ceph-mon[21867]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T21:12:41.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:41 smithi130 ceph-mon[26394]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T21:12:43.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:43 smithi150 ceph-mon[22676]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 963 B/s rd, 0 op/s 2024-04-22T21:12:43.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:43 smithi130 ceph-mon[21867]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 963 B/s rd, 0 op/s 2024-04-22T21:12:43.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:43 smithi130 ceph-mon[26394]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 963 B/s rd, 0 op/s 2024-04-22T21:12:45.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:45 smithi130 ceph-mon[21867]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:45.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:45 smithi130 ceph-mon[26394]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:45.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:45 smithi150 ceph-mon[22676]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:46.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:46 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:12:46.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:46 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:12:46.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:46 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:12:46.859 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:12:46 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:12:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:12:47.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:47 smithi130 ceph-mon[26394]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:47.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:47 smithi130 ceph-mon[21867]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:47.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:47 smithi150 ceph-mon[22676]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:49.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:49 smithi150 ceph-mon[22676]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:49.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:49 smithi130 ceph-mon[26394]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:49.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:49 smithi130 ceph-mon[21867]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:50.306 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:12:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:12:49] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-04-22T21:12:51.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:51 smithi150 ceph-mon[22676]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:51.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:51 smithi130 ceph-mon[21867]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:51.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:51 smithi130 ceph-mon[26394]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:53.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:53 smithi150 ceph-mon[22676]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:53.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:53 smithi130 ceph-mon[21867]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:53.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:53 smithi130 ceph-mon[26394]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:55.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:55 smithi130 ceph-mon[21867]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:55.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:55 smithi130 ceph-mon[26394]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:55.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:55 smithi150 ceph-mon[22676]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:56.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:56 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:12:56.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:56 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:12:56.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:56 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:12:56.859 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:12:56 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:12:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:12:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:57 smithi130 ceph-mon[21867]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:57.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:57 smithi130 ceph-mon[26394]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:57.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:57 smithi150 ceph-mon[22676]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:12:59.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:12:59 smithi150 ceph-mon[22676]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:59.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:12:59 smithi130 ceph-mon[21867]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:12:59.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:12:59 smithi130 ceph-mon[26394]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:00.305 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:12:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:12:59] "GET /metrics HTTP/1.1" 200 207957 "" "Prometheus/2.33.4" 2024-04-22T21:13:01.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:01 smithi150 ceph-mon[22676]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:01.478 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:01 smithi130 ceph-mon[26394]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:01.479 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:01 smithi130 ceph-mon[21867]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:03.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:03 smithi150 ceph-mon[22676]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:03.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:03 smithi130 ceph-mon[26394]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:03.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:03 smithi130 ceph-mon[21867]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:05.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:05 smithi130 ceph-mon[21867]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:05.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:05 smithi130 ceph-mon[26394]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:05.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:05 smithi150 ceph-mon[22676]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:06.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:06 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:06.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:06 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:06.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:06 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:06.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:13:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:13:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:13:07.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:07 smithi130 ceph-mon[26394]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:07.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:07 smithi130 ceph-mon[21867]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:07.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:07 smithi150 ceph-mon[22676]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:09.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:09 smithi150 ceph-mon[22676]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:09.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:09 smithi130 ceph-mon[21867]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:09.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:09 smithi130 ceph-mon[26394]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:10.305 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:13:09 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:13:09] "GET /metrics HTTP/1.1" 200 207957 "" "Prometheus/2.33.4" 2024-04-22T21:13:10.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:13:10.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:13:10.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:13:10.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:13:10.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:13:10.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:13:11.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:11 smithi150 ceph-mon[22676]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:11.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:11 smithi130 ceph-mon[21867]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:11.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:11 smithi130 ceph-mon[26394]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:13.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:13 smithi150 ceph-mon[22676]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:13.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:13 smithi130 ceph-mon[21867]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:13.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:13 smithi130 ceph-mon[26394]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:15.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:15 smithi130 ceph-mon[21867]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:15.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:15 smithi130 ceph-mon[26394]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:15.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:15 smithi150 ceph-mon[22676]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:16.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:16 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:16.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:16 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:16.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:16 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:16.858 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:13:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:13:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:13:17.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:17 smithi130 ceph-mon[21867]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:17.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:17 smithi130 ceph-mon[26394]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:17.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:17 smithi150 ceph-mon[22676]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:19.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:19 smithi150 ceph-mon[22676]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:19.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:13:19.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:13:19.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:13:19.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:19 smithi130 ceph-mon[21867]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:19.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:13:19.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:13:19.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:13:19.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:19 smithi130 ceph-mon[26394]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:19.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:13:19.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:13:19.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:13:20.305 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:13:19 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:13:19] "GET /metrics HTTP/1.1" 200 207932 "" "Prometheus/2.33.4" 2024-04-22T21:13:21.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:21 smithi150 ceph-mon[22676]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:21.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:21 smithi130 ceph-mon[21867]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:21.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:21 smithi130 ceph-mon[26394]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:23.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:23 smithi150 ceph-mon[22676]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:23.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:23 smithi130 ceph-mon[26394]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:23.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:23 smithi130 ceph-mon[21867]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:25.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:25 smithi130 ceph-mon[26394]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:25.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:25 smithi130 ceph-mon[21867]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:25.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:25 smithi150 ceph-mon[22676]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:26.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:26 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:26.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:26 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:26.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:26 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:26.858 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:13:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:13:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:13:27.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:27 smithi130 ceph-mon[21867]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:27.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:27 smithi130 ceph-mon[26394]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:27.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:27 smithi150 ceph-mon[22676]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:29.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:29 smithi150 ceph-mon[22676]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:29.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:29 smithi130 ceph-mon[21867]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:29.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:29 smithi130 ceph-mon[26394]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:30.304 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:13:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:13:29] "GET /metrics HTTP/1.1" 200 207941 "" "Prometheus/2.33.4" 2024-04-22T21:13:31.361 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:31 smithi150 ceph-mon[22676]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:31.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:31 smithi130 ceph-mon[21867]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:31.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:31 smithi130 ceph-mon[26394]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:33.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:33 smithi150 ceph-mon[22676]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:33.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:33 smithi130 ceph-mon[21867]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:33.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:33 smithi130 ceph-mon[26394]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:35.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:35 smithi130 ceph-mon[21867]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:35.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:35 smithi130 ceph-mon[26394]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:35.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:35 smithi150 ceph-mon[22676]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:36.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:36 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:36.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:36 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:36.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:36 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:36.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:13:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:13:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:13:37.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:37 smithi130 ceph-mon[21867]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:37.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:37 smithi130 ceph-mon[26394]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:37.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:37 smithi150 ceph-mon[22676]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:39.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:39 smithi150 ceph-mon[22676]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:39.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:39 smithi130 ceph-mon[21867]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:39.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:39 smithi130 ceph-mon[26394]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:40.304 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:13:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:13:39] "GET /metrics HTTP/1.1" 200 207941 "" "Prometheus/2.33.4" 2024-04-22T21:13:41.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:41 smithi150 ceph-mon[22676]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:41.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:41 smithi130 ceph-mon[21867]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:41.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:41 smithi130 ceph-mon[26394]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:43.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:43 smithi150 ceph-mon[22676]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:43.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:43 smithi130 ceph-mon[21867]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:43.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:43 smithi130 ceph-mon[26394]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:45.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:45 smithi130 ceph-mon[21867]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:45.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:45 smithi130 ceph-mon[26394]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:45.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:45 smithi150 ceph-mon[22676]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:46.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:46 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:46.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:46 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:46.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:46 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:46.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:13:46 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:13:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:13:47.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:47 smithi130 ceph-mon[26394]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:47.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:47 smithi130 ceph-mon[21867]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:47.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:47 smithi150 ceph-mon[22676]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:49.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:49 smithi150 ceph-mon[22676]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:49.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:49 smithi130 ceph-mon[21867]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:49.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:49 smithi130 ceph-mon[26394]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:50.304 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:13:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:13:49] "GET /metrics HTTP/1.1" 200 207938 "" "Prometheus/2.33.4" 2024-04-22T21:13:51.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:51 smithi150 ceph-mon[22676]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:51.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:51 smithi130 ceph-mon[21867]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:51.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:51 smithi130 ceph-mon[26394]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:53.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:53 smithi150 ceph-mon[22676]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:53.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:53 smithi130 ceph-mon[21867]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:53.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:53 smithi130 ceph-mon[26394]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:55.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:55 smithi130 ceph-mon[21867]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:55.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:55 smithi130 ceph-mon[26394]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:55.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:55 smithi150 ceph-mon[22676]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:56.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:56 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:56.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:56 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:56.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:56 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:13:56.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:13:56 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:13:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:13:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:57 smithi130 ceph-mon[21867]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:57.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:57 smithi130 ceph-mon[26394]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:57.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:57 smithi150 ceph-mon[22676]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:13:59.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:13:59 smithi150 ceph-mon[22676]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:59.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:13:59 smithi130 ceph-mon[21867]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:13:59.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:13:59 smithi130 ceph-mon[26394]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:00.304 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:13:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:13:59] "GET /metrics HTTP/1.1" 200 207940 "" "Prometheus/2.33.4" 2024-04-22T21:14:01.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:01 smithi130 ceph-mon[21867]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:01.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:01 smithi130 ceph-mon[26394]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:01.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:01 smithi150 ceph-mon[22676]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:03.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:03 smithi130 ceph-mon[21867]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:03.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:03 smithi130 ceph-mon[26394]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:03.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:03 smithi150 ceph-mon[22676]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:05.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:05 smithi130 ceph-mon[21867]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:05.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:05 smithi130 ceph-mon[26394]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:05.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:05 smithi150 ceph-mon[22676]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:06.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:06 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:06.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:06 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:06.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:06 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:06.858 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:14:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:14:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:14:07.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:07 smithi130 ceph-mon[21867]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:07.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:07 smithi130 ceph-mon[26394]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:07.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:07 smithi150 ceph-mon[22676]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:09.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:09 smithi150 ceph-mon[22676]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:09.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:09 smithi130 ceph-mon[21867]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:09.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:09 smithi130 ceph-mon[26394]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:10.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:14:10.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:14:10.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:14:10.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:14:10.305 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:14:09 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:14:09] "GET /metrics HTTP/1.1" 200 207940 "" "Prometheus/2.33.4" 2024-04-22T21:14:10.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:14:10.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:14:11.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:11 smithi130 ceph-mon[21867]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:14:11.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:11 smithi130 ceph-mon[26394]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:14:11.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:11 smithi150 ceph-mon[22676]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:14:13.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:13 smithi130 ceph-mon[21867]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:13.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:13 smithi130 ceph-mon[26394]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:13.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:13 smithi150 ceph-mon[22676]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:15.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:15 smithi130 ceph-mon[21867]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:14:15.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:15 smithi130 ceph-mon[26394]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:14:15.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:15 smithi150 ceph-mon[22676]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:14:16.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:16 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:16.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:16 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:16.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:16 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:16.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:14:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:14:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:14:17.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:17 smithi130 ceph-mon[26394]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:17.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:17 smithi130 ceph-mon[21867]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:17.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:17 smithi150 ceph-mon[22676]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:19.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:19 smithi130 ceph-mon[26394]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:14:19.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:14:19.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:19 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:14:19.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:19 smithi130 ceph-mon[21867]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:14:19.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:14:19.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:19 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:14:19.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:19 smithi150 ceph-mon[22676]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:14:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:14:19.610 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:19 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:14:20.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:20 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:14:20.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:20 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:14:20.304 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:14:19 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:14:19] "GET /metrics HTTP/1.1" 200 207945 "" "Prometheus/2.33.4" 2024-04-22T21:14:20.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:20 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:14:21.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:21 smithi130 ceph-mon[26394]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:14:21.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:21 smithi130 ceph-mon[21867]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:14:21.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:21 smithi150 ceph-mon[22676]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:14:23.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:23 smithi130 ceph-mon[26394]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:23.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:23 smithi130 ceph-mon[21867]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:23.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:23 smithi150 ceph-mon[22676]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:25.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:25 smithi130 ceph-mon[21867]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:14:25.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:25 smithi130 ceph-mon[26394]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:14:25.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:25 smithi150 ceph-mon[22676]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:14:26.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:26 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:26.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:26 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:26.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:26 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:26.607 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:14:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:14:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:14:27.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:27 smithi130 ceph-mon[21867]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:27.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:27 smithi130 ceph-mon[26394]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:27.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:27 smithi150 ceph-mon[22676]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:29.444 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:29 smithi130 ceph-mon[21867]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:29.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:29 smithi130 ceph-mon[26394]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:29.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:29 smithi150 ceph-mon[22676]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:30.304 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:14:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:14:29] "GET /metrics HTTP/1.1" 200 207930 "" "Prometheus/2.33.4" 2024-04-22T21:14:31.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:31 smithi130 ceph-mon[21867]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:31.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:31 smithi130 ceph-mon[26394]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:31.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:31 smithi150 ceph-mon[22676]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:33.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:33 smithi130 ceph-mon[21867]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:33.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:33 smithi130 ceph-mon[26394]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:33.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:33 smithi150 ceph-mon[22676]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:35.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:35 smithi130 ceph-mon[26394]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:35.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:35 smithi130 ceph-mon[21867]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:35.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:35 smithi150 ceph-mon[22676]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:36.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:36 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:36.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:36 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:36.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:36 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:36.857 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:14:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:14:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:14:37.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:37 smithi130 ceph-mon[21867]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:37.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:37 smithi130 ceph-mon[26394]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:37.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:37 smithi150 ceph-mon[22676]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:39.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:39 smithi130 ceph-mon[21867]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:39.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:39 smithi130 ceph-mon[26394]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:39.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:39 smithi150 ceph-mon[22676]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:40.304 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:14:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:14:39] "GET /metrics HTTP/1.1" 200 207930 "" "Prometheus/2.33.4" 2024-04-22T21:14:41.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:41 smithi130 ceph-mon[21867]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:41.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:41 smithi130 ceph-mon[26394]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:41.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:41 smithi150 ceph-mon[22676]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:43.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:43 smithi130 ceph-mon[21867]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:43.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:43 smithi130 ceph-mon[26394]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:43.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:43 smithi150 ceph-mon[22676]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:45.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:45 smithi130 ceph-mon[26394]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:45.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:45 smithi130 ceph-mon[21867]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:45.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:45 smithi150 ceph-mon[22676]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:46.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:46 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:46.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:46 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:46.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:46 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:46.607 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:14:46 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:14:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:14:47.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:47 smithi130 ceph-mon[26394]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:47.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:47 smithi130 ceph-mon[21867]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:47.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:47 smithi150 ceph-mon[22676]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:49.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:49 smithi130 ceph-mon[21867]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:49.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:49 smithi130 ceph-mon[26394]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:49.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:49 smithi150 ceph-mon[22676]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:50.304 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:14:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:14:49] "GET /metrics HTTP/1.1" 200 207890 "" "Prometheus/2.33.4" 2024-04-22T21:14:51.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:51 smithi130 ceph-mon[21867]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:51.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:51 smithi130 ceph-mon[26394]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:51.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:51 smithi150 ceph-mon[22676]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:53.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:53 smithi130 ceph-mon[21867]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:53.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:53 smithi130 ceph-mon[26394]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:53.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:53 smithi150 ceph-mon[22676]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:55.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:55 smithi130 ceph-mon[26394]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:55.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:55 smithi130 ceph-mon[21867]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:55.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:55 smithi150 ceph-mon[22676]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:56.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:56 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:56.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:56 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:56.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:56 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:14:56.857 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:14:56 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:14:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:14:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:57 smithi130 ceph-mon[21867]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:57.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:57 smithi130 ceph-mon[26394]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:57.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:57 smithi150 ceph-mon[22676]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:14:59.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:14:59 smithi130 ceph-mon[21867]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:59.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:14:59 smithi130 ceph-mon[26394]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:14:59.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:14:59 smithi150 ceph-mon[22676]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:00.303 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:14:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:14:59] "GET /metrics HTTP/1.1" 200 207884 "" "Prometheus/2.33.4" 2024-04-22T21:15:01.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:01 smithi130 ceph-mon[21867]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:01.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:01 smithi130 ceph-mon[26394]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:01.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:01 smithi150 ceph-mon[22676]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:03.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:03 smithi130 ceph-mon[21867]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:03.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:03 smithi130 ceph-mon[26394]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:03.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:03 smithi150 ceph-mon[22676]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:05.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:05 smithi130 ceph-mon[21867]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:05.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:05 smithi130 ceph-mon[26394]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:05.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:05 smithi150 ceph-mon[22676]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:06.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:06 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:06.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:06 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:06.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:06 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:06.607 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:15:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:15:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:15:07.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:07 smithi130 ceph-mon[21867]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:07.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:07 smithi130 ceph-mon[26394]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:07.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:07 smithi150 ceph-mon[22676]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:09.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:09 smithi130 ceph-mon[21867]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:09.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:09 smithi130 ceph-mon[26394]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:09.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:09 smithi150 ceph-mon[22676]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:10.132 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:15:09 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:15:09] "GET /metrics HTTP/1.1" 200 207884 "" "Prometheus/2.33.4" 2024-04-22T21:15:10.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:15:10.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:15:10.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:15:10.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:15:10.597 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-22T21:15:10.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:15:10.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:15:11.229 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:11 smithi130 ceph-mon[21867]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:11.229 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:11 smithi130 ceph-mon[26394]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:11.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:11 smithi150 ceph-mon[22676]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:13.165 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-22T21:15:13.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:13 smithi130 ceph-mon[21867]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:13.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:13 smithi130 ceph-mon[26394]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:13.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:13 smithi150 ceph-mon[22676]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:15.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:15 smithi130 ceph-mon[21867]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:15.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:15 smithi130 ceph-mon[26394]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:15.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:15 smithi150 ceph-mon[22676]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:15.693 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-22T21:15:16.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:16 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:16.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:16 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:16.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:16 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:16.857 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:15:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:15:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:15:17.238 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:17 smithi130 ceph-mon[26394]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:17.238 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:17 smithi130 ceph-mon[21867]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:17.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:17 smithi150 ceph-mon[22676]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:18.365 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps' 2024-04-22T21:15:19.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:19 smithi130 ceph-mon[21867]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:19.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:19 smithi130 ceph-mon[26394]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:19.447 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:19 smithi150 ceph-mon[22676]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:20.007 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:15:19 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:15:19] "GET /metrics HTTP/1.1" 200 207888 "" "Prometheus/2.33.4" 2024-04-22T21:15:20.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:20 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:15:20.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:20 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:15:20.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:20 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:15:20.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:20 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:15:20.425 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:15:20.426 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (3m) 3m ago 3m 15.5M - ba2b418f427c 2176cd6f1d6a 2024-04-22T21:15:20.426 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (3m) 3m ago 3m 45.1M - 8.3.5 dad864ee21e9 b3f3edd15758 2024-04-22T21:15:20.426 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (3m) 3m ago 3m 69.4M - 3.5 e1d6a67b021e 264d4f175cde 2024-04-22T21:15:20.426 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e 2c3737817897 2024-04-22T21:15:20.426 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e 878d76986ae8 2024-04-22T21:15:20.426 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (10m) 3m ago 10m 48.8M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:15:20.426 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (8m) 3m ago 8m 39.0M 2048M 17.2.0 e1d6a67b021e 861909a68cf9 2024-04-22T21:15:20.426 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (9m) 3m ago 9m 39.2M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:15:20.426 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (4m) 3m ago 4m 14.5M - 1dbe0e931976 82b1ae40160e 2024-04-22T21:15:20.426 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (4m) 3m ago 4m 16.6M - 1dbe0e931976 aafb83793157 2024-04-22T21:15:20.426 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (8m) 3m ago 8m 47.8M 4016M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:15:20.426 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (8m) 3m ago 7m 50.7M 4016M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:15:20.426 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (7m) 3m ago 7m 44.6M 4016M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:15:20.427 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (7m) 3m ago 7m 45.4M 4016M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:15:20.427 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (6m) 3m ago 6m 45.7M 4272M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:15:20.427 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (6m) 3m ago 6m 44.4M 4272M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:15:20.427 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (6m) 3m ago 6m 45.8M 4272M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:15:20.427 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (5m) 3m ago 5m 45.6M 4272M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:15:20.427 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (3m) 3m ago 4m 42.4M - 514e6a882f6e afc570412b78 2024-04-22T21:15:20.427 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (3m) 3m ago 3m 70.8M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:15:20.427 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (3m) 3m ago 3m 68.5M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:15:20.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:20 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:15:20.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:20 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:15:21.223 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions' 2024-04-22T21:15:21.530 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:21 smithi130 ceph-mon[21867]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:21.531 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:21 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:15:21.531 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:21 smithi130 ceph-mon[26394]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:21.531 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:21 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:15:21.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:21 smithi150 ceph-mon[22676]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:21.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:21 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:15:22.486 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:22 smithi130 ceph-mon[26394]: from='client.24871 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:15:22.486 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:22 smithi130 ceph-mon[21867]: from='client.24871 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:15:22.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:22 smithi150 ceph-mon[22676]: from='client.24871 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:15:23.164 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:15:23.165 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:15:23.165 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T21:15:23.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:15:23.165 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:15:23.165 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:15:23.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:15:23.165 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:15:23.165 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:15:23.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:15:23.165 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": {}, 2024-04-22T21:15:23.165 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:15:23.166 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:15:23.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:15:23.166 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:15:23.166 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-04-22T21:15:23.166 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:15:23.166 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:15:23.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:23 smithi130 ceph-mon[21867]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:23.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:23 smithi130 ceph-mon[26394]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:23.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:23 smithi150 ceph-mon[22676]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:23.875 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph -s' 2024-04-22T21:15:24.428 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:24 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/653593509' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:15:24.429 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:24 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/653593509' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:15:24.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:24 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/653593509' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:15:25.165 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:25 smithi130 ceph-mon[21867]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:25.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:25 smithi130 ceph-mon[26394]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:25.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:25 smithi150 ceph-mon[22676]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:25.900 INFO:teuthology.orchestra.run.smithi130.stdout: cluster: 2024-04-22T21:15:25.901 INFO:teuthology.orchestra.run.smithi130.stdout: id: dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:15:25.901 INFO:teuthology.orchestra.run.smithi130.stdout: health: HEALTH_OK 2024-04-22T21:15:25.901 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:15:25.901 INFO:teuthology.orchestra.run.smithi130.stdout: services: 2024-04-22T21:15:25.901 INFO:teuthology.orchestra.run.smithi130.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-04-22T21:15:25.901 INFO:teuthology.orchestra.run.smithi130.stdout: mgr: y(active, since 5m), standbys: x 2024-04-22T21:15:25.901 INFO:teuthology.orchestra.run.smithi130.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-04-22T21:15:25.901 INFO:teuthology.orchestra.run.smithi130.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-22T21:15:25.901 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:15:25.901 INFO:teuthology.orchestra.run.smithi130.stdout: data: 2024-04-22T21:15:25.901 INFO:teuthology.orchestra.run.smithi130.stdout: pools: 6 pools, 161 pgs 2024-04-22T21:15:25.901 INFO:teuthology.orchestra.run.smithi130.stdout: objects: 209 objects, 585 KiB 2024-04-22T21:15:25.901 INFO:teuthology.orchestra.run.smithi130.stdout: usage: 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:15:25.902 INFO:teuthology.orchestra.run.smithi130.stdout: pgs: 161 active+clean 2024-04-22T21:15:25.902 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:15:25.902 INFO:teuthology.orchestra.run.smithi130.stdout: io: 2024-04-22T21:15:25.902 INFO:teuthology.orchestra.run.smithi130.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-22T21:15:25.902 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:15:26.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:26 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:26.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:26 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/913334730' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:15:26.490 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:26 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:26.490 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:26 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/913334730' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:15:26.512 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ls' 2024-04-22T21:15:26.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:26 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:26.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:26 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/913334730' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:15:26.607 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:15:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:15:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:15:27.454 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:27 smithi130 ceph-mon[21867]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:27.454 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:27 smithi130 ceph-mon[26394]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:27.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:27 smithi150 ceph-mon[22676]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:28.496 INFO:teuthology.orchestra.run.smithi130.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-22T21:15:28.496 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi130=a;count:1 2024-04-22T21:15:28.496 INFO:teuthology.orchestra.run.smithi130.stdout:grafana ?:3000 1/1 3m ago 5m smithi150=a;count:1 2024-04-22T21:15:28.496 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-04-22T21:15:28.497 INFO:teuthology.orchestra.run.smithi130.stdout:mgr 2/2 3m ago 8m smithi130=y;smithi150=x;count:2 2024-04-22T21:15:28.497 INFO:teuthology.orchestra.run.smithi130.stdout:mon 3/3 3m ago 9m smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi150:172.21.15.150=b;count:3 2024-04-22T21:15:28.497 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi130=a;smithi150=b;count:2 2024-04-22T21:15:28.497 INFO:teuthology.orchestra.run.smithi130.stdout:osd 8 3m ago - 2024-04-22T21:15:28.497 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus ?:9095 1/1 3m ago 5m smithi150=a;count:1 2024-04-22T21:15:28.497 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-04-22T21:15:29.227 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-22T21:15:29.534 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:29 smithi130 ceph-mon[26394]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:29.535 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:29 smithi130 ceph-mon[21867]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:29.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:29 smithi150 ceph-mon[22676]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:30.161 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:15:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:15:29] "GET /metrics HTTP/1.1" 200 207875 "" "Prometheus/2.33.4" 2024-04-22T21:15:30.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:30 smithi130 ceph-mon[21867]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:15:30.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:30 smithi130 ceph-mon[26394]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:15:30.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:30 smithi150 ceph-mon[22676]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:15:31.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:31 smithi130 ceph-mon[26394]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:31.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:31 smithi130 ceph-mon[21867]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:31.596 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled to redeploy mgr.x on host 'smithi150' 2024-04-22T21:15:31.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:31 smithi150 ceph-mon[22676]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:32.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/938880219' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T21:15:32.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:15:32.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:15:32.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:15:32.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:15:32.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:15:32.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:15:32.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:15:32.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/938880219' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T21:15:32.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:15:32.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:15:32.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:15:32.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:15:32.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:15:32.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:15:32.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:32 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:15:32.316 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps --refresh' 2024-04-22T21:15:32.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:32 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/938880219' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T21:15:32.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:32 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:15:32.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:32 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:15:32.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:32 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:15:32.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:32 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:15:32.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:32 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:15:32.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:32 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:15:32.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:32 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:15:33.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:33 smithi130 ceph-mon[26394]: from='client.15009 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:15:33.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:33 smithi130 ceph-mon[26394]: Schedule redeploy daemon mgr.x 2024-04-22T21:15:33.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:33 smithi130 ceph-mon[26394]: Deploying daemon mgr.x on smithi150 2024-04-22T21:15:33.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:33 smithi130 ceph-mon[26394]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:33.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:33 smithi130 ceph-mon[21867]: from='client.15009 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:15:33.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:33 smithi130 ceph-mon[21867]: Schedule redeploy daemon mgr.x 2024-04-22T21:15:33.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:33 smithi130 ceph-mon[21867]: Deploying daemon mgr.x on smithi150 2024-04-22T21:15:33.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:33 smithi130 ceph-mon[21867]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:33.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:33 smithi150 ceph-mon[22676]: from='client.15009 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:15:33.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:33 smithi150 ceph-mon[22676]: Schedule redeploy daemon mgr.x 2024-04-22T21:15:33.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:33 smithi150 ceph-mon[22676]: Deploying daemon mgr.x on smithi150 2024-04-22T21:15:33.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:33 smithi150 ceph-mon[22676]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:34.318 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:15:34.318 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (3m) 3m ago 4m 15.5M - ba2b418f427c 2176cd6f1d6a 2024-04-22T21:15:34.318 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (3m) 3m ago 3m 45.1M - 8.3.5 dad864ee21e9 b3f3edd15758 2024-04-22T21:15:34.318 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (3m) 3m ago 3m 69.4M - 3.5 e1d6a67b021e 264d4f175cde 2024-04-22T21:15:34.318 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e 2c3737817897 2024-04-22T21:15:34.318 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e 878d76986ae8 2024-04-22T21:15:34.318 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (10m) 3m ago 10m 48.8M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:15:34.318 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (9m) 3m ago 9m 39.0M 2048M 17.2.0 e1d6a67b021e 861909a68cf9 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (9m) 3m ago 9m 39.2M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (5m) 3m ago 5m 14.5M - 1dbe0e931976 82b1ae40160e 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (4m) 3m ago 4m 16.6M - 1dbe0e931976 aafb83793157 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (8m) 3m ago 8m 47.8M 4016M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (8m) 3m ago 8m 50.7M 4016M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (7m) 3m ago 7m 44.6M 4016M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (7m) 3m ago 7m 45.4M 4016M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (7m) 3m ago 7m 45.7M 4272M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (6m) 3m ago 6m 44.4M 4272M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (6m) 3m ago 6m 45.8M 4272M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (5m) 3m ago 5m 45.6M 4272M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (3m) 3m ago 5m 42.4M - 514e6a882f6e afc570412b78 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (3m) 3m ago 3m 70.8M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:15:34.319 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (3m) 3m ago 3m 68.5M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:15:35.107 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'sleep 180' 2024-04-22T21:15:35.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:35 smithi130 ceph-mon[21867]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:35.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:35 smithi130 ceph-mon[26394]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:35.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:35 smithi150 ceph-mon[22676]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:36.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:36 smithi130 ceph-mon[26394]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:15:36.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:36 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:36.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:36 smithi130 ceph-mon[21867]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:15:36.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:36 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:36.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:36 smithi150 ceph-mon[22676]: from='client.15015 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:15:36.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:36 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:36.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:15:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:15:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:15:37.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:37 smithi130 ceph-mon[21867]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:37.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:37 smithi130 ceph-mon[26394]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:37.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:37 smithi150 ceph-mon[22676]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:39.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:39 smithi130 ceph-mon[26394]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:39.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:39 smithi130 ceph-mon[21867]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:39.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:39 smithi150 ceph-mon[22676]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:40.303 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:15:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:15:39] "GET /metrics HTTP/1.1" 200 207875 "" "Prometheus/2.33.4" 2024-04-22T21:15:41.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:41 smithi130 ceph-mon[26394]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:41.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:41 smithi130 ceph-mon[21867]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:41.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:41 smithi150 ceph-mon[22676]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:43.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:43 smithi130 ceph-mon[26394]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:43.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:43 smithi130 ceph-mon[21867]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:43.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:43 smithi150 ceph-mon[22676]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:45.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:45 smithi130 ceph-mon[26394]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:45.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:45 smithi130 ceph-mon[21867]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:45.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:45 smithi150 ceph-mon[22676]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:46.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:46 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:46.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:46 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:46.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:46 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:46.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:15:46 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:15:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:15:47.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:47 smithi150 ceph-mon[22676]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:47.453 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:47 smithi130 ceph-mon[26394]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:47.454 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:47 smithi130 ceph-mon[21867]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:49.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:49 smithi130 ceph-mon[26394]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:49.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:49 smithi130 ceph-mon[21867]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:49.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:49 smithi150 ceph-mon[22676]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:50.306 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:15:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:15:49] "GET /metrics HTTP/1.1" 200 207889 "" "Prometheus/2.33.4" 2024-04-22T21:15:51.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:51 smithi130 ceph-mon[26394]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:51.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:51 smithi130 ceph-mon[21867]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:51.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:51 smithi150 ceph-mon[22676]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:53.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:53 smithi130 ceph-mon[26394]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:53.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:53 smithi130 ceph-mon[21867]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:53.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:53 smithi150 ceph-mon[22676]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:55.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:55 smithi130 ceph-mon[21867]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:55.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:55 smithi130 ceph-mon[26394]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:55.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:55 smithi150 ceph-mon[22676]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:56.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:56 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:56.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:56 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:56 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:15:56.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:15:56 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[23852]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:15:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:15:57.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:57 smithi150 ceph-mon[22676]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:57.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:57 smithi130 ceph-mon[26394]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:57 smithi130 ceph-mon[21867]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:15:59.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:15:59 smithi130 ceph-mon[21867]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:59.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:15:59 smithi130 ceph-mon[26394]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:15:59.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:15:59 smithi150 ceph-mon[22676]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:00.303 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:15:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:15:59] "GET /metrics HTTP/1.1" 200 207883 "" "Prometheus/2.33.4" 2024-04-22T21:16:01.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:01 smithi130 ceph-mon[21867]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:01.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:01 smithi130 ceph-mon[26394]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:01.571 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:01 smithi150 ceph-mon[22676]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:03.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:03 smithi150 ceph-mon[22676]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:03.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:03 smithi130 ceph-mon[21867]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:03.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:03 smithi130 ceph-mon[26394]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:04.857 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:04 smithi150 systemd[1]: Stopping Ceph mgr.x for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:16:04.857 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:04 smithi150 bash[42240]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr.x" found: no such container 2024-04-22T21:16:04.857 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:04 smithi150 podman[42252]: 2024-04-22 21:16:04.595137329 +0000 UTC m=+0.142983971 container died 2c373781789782e87c284a96c7c4c2dfd8ba357f1b3d942146247bdb69eb5055 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, GIT_CLEAN=True, version=8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, release=754, vcs-type=git, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, GIT_BRANCH=HEAD) 2024-04-22T21:16:05.164 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:05 smithi150 ceph-mon[22676]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:05.165 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:04 smithi150 podman[42252]: 2024-04-22 21:16:04.974484695 +0000 UTC m=+0.522331336 container cleanup 2c373781789782e87c284a96c7c4c2dfd8ba357f1b3d942146247bdb69eb5055 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, io.buildah.version=1.19.8, ceph=True, GIT_CLEAN=True, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, RELEASE=HEAD, vcs-type=git, maintainer=Guillaume Abrioux , name=centos-stream, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-22T21:16:05.165 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:04 smithi150 bash[42252]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x 2024-04-22T21:16:05.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:05 smithi130 ceph-mon[21867]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:05.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:05 smithi130 ceph-mon[26394]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:05.422 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 podman[42266]: 2024-04-22 21:16:05.175802042 +0000 UTC m=+0.577278573 container remove 2c373781789782e87c284a96c7c4c2dfd8ba357f1b3d942146247bdb69eb5055 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_CLEAN=True, distribution-scope=public, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.buildah.version=1.19.8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, release=754) 2024-04-22T21:16:05.422 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-22T21:16:05.422 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 bash[42280]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr.x" found: no such container 2024-04-22T21:16:05.423 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-22T21:16:05.423 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 systemd[1]: Stopped Ceph mgr.x for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:16:05.423 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.x.service: Consumed 17.167s CPU time. 2024-04-22T21:16:05.754 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 systemd[1]: Starting Ceph mgr.x for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:16:05.754 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 podman[42386]: 2024-04-22 21:16:05.554111972 +0000 UTC m=+0.017600354 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:16:05.754 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 podman[42386]: 2024-04-22 21:16:05.660140644 +0000 UTC m=+0.123629013 container create 327605fe868fbd6f31a6a1d7799d6f873fbb9b77ca5f0915b277c808e6ee11eb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, GIT_CLEAN=True, ceph=True) 2024-04-22T21:16:06.032 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 podman[42386]: 2024-04-22 21:16:05.887845897 +0000 UTC m=+0.351334266 container init 327605fe868fbd6f31a6a1d7799d6f873fbb9b77ca5f0915b277c808e6ee11eb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240416) 2024-04-22T21:16:06.032 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 podman[42386]: 2024-04-22 21:16:05.890445919 +0000 UTC m=+0.353934289 container start 327605fe868fbd6f31a6a1d7799d6f873fbb9b77ca5f0915b277c808e6ee11eb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:16:06.032 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 ceph-mgr[42406]: -- 172.21.15.150:0/3834082448 <== mon.2 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55821af385a0 con 0x55821af0c800 2024-04-22T21:16:06.032 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 bash[42386]: 327605fe868fbd6f31a6a1d7799d6f873fbb9b77ca5f0915b277c808e6ee11eb 2024-04-22T21:16:06.033 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:05 smithi150 systemd[1]: Started Ceph mgr.x for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:16:06.284 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:06.041+0000 7f40a3e76200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:16:06.284 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:06.236+0000 7f40a3e76200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:16:06.285 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:06 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:06.285 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:06 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:06.285 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:06 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:06.285 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:06 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:16:06.285 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:06 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:16:06.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:06 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:06.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:06 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:06.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:06 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:06.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:06 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:16:06.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:06 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:16:06.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:06 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:06.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:06 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:06.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:06 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:06.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:06 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:16:06.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:06 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:16:06.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:06.353+0000 7f40a3e76200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:16:06.992 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:06.692+0000 7f40a3e76200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:16:06.992 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:06.777+0000 7f40a3e76200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:16:07.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:07 smithi150 ceph-mon[22676]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:07.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:07 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:07.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:07 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:07.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:07 smithi130 ceph-mon[26394]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:07.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:07 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:07.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:07 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:07.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:07 smithi130 ceph-mon[21867]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:07.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:07 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:07.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:07 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:08.356 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:08.042+0000 7f40a3e76200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:16:08.357 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:08.267+0000 7f40a3e76200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:16:08.729 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:08.452+0000 7f40a3e76200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:16:09.052 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:08.758+0000 7f40a3e76200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:16:09.052 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:08.852+0000 7f40a3e76200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:16:09.317 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:09 smithi150 ceph-mon[22676]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:09.317 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:09.061+0000 7f40a3e76200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:16:09.317 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:09.268+0000 7f40a3e76200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:16:09.395 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:09 smithi130 ceph-mon[26394]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:09.396 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:09 smithi130 ceph-mon[21867]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:10.255 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:09.897+0000 7f40a3e76200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:16:10.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:16:09 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:16:09] "GET /metrics HTTP/1.1" 200 207883 "" "Prometheus/2.33.4" 2024-04-22T21:16:10.507 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:10.333+0000 7f40a3e76200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:16:10.507 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:10.433+0000 7f40a3e76200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:16:10.789 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:10.789 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:16:10.789 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:16:10.789 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:10 smithi150 ceph-mon[22676]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:10.789 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:10.639+0000 7f40a3e76200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T21:16:11.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:11.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:16:11.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:16:11.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:10 smithi130 ceph-mon[26394]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:11.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:11.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:16:11.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:16:11.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:10 smithi130 ceph-mon[21867]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:11.506 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:11.173+0000 7f40a3e76200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:16:11.506 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:11.267+0000 7f40a3e76200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:16:11.801 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:11.515+0000 7f40a3e76200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:16:11.801 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:11.650+0000 7f40a3e76200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:16:11.801 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:11.746+0000 7f40a3e76200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:16:12.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:12.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:12.053 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:12.053 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:12.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:11.862+0000 7f40a3e76200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:16:12.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:12.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:16:12.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:12.167+0000 7f40a3e76200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:16:13.090 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:12 smithi150 ceph-mon[22676]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:13.091 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:12.774+0000 7f40a3e76200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:16:13.091 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:12.993+0000 7f40a3e76200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:16:13.091 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:16:12] ENGINE Bus STARTING 2024-04-22T21:16:13.091 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: CherryPy Checker: 2024-04-22T21:16:13.091 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: The Application mounted at '' has an empty config. 2024-04-22T21:16:13.091 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:16:13.091 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:16:13] ENGINE Serving on http://:::9283 2024-04-22T21:16:13.091 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:16:13] ENGINE Bus STARTED 2024-04-22T21:16:13.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:12 smithi130 ceph-mon[26394]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:13.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:12 smithi130 ceph-mon[21867]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:14.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:13 smithi150 ceph-mon[22676]: Standby manager daemon x restarted 2024-04-22T21:16:14.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:13 smithi150 ceph-mon[22676]: Standby manager daemon x started 2024-04-22T21:16:14.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:13 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:16:14.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:13 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:16:14.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:13 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:16:14.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:13 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:16:14.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:13 smithi130 ceph-mon[21867]: Standby manager daemon x restarted 2024-04-22T21:16:14.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:13 smithi130 ceph-mon[21867]: Standby manager daemon x started 2024-04-22T21:16:14.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:13 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:16:14.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:13 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:16:14.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:13 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:16:14.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:13 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:16:14.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:13 smithi130 ceph-mon[26394]: Standby manager daemon x restarted 2024-04-22T21:16:14.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:13 smithi130 ceph-mon[26394]: Standby manager daemon x started 2024-04-22T21:16:14.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:13 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:16:14.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:13 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:16:14.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:13 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:16:14.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:13 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:16:15.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:14 smithi150 ceph-mon[22676]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-22T21:16:15.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:14 smithi150 ceph-mon[22676]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:15.302 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:14 smithi130 ceph-mon[26394]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-22T21:16:15.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:14 smithi130 ceph-mon[26394]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:15.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:14 smithi130 ceph-mon[21867]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-22T21:16:15.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:14 smithi130 ceph-mon[21867]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:16.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:16 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:16 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:16.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:16 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:16.856 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:16:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:16:17.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:17 smithi150 ceph-mon[22676]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:17.454 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:17 smithi130 ceph-mon[21867]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:17.454 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:17 smithi130 ceph-mon[26394]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:19.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:19 smithi130 ceph-mon[21867]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:19.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:19 smithi130 ceph-mon[26394]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:19.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:19 smithi150 ceph-mon[22676]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:20.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:16:19 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:16:19] "GET /metrics HTTP/1.1" 200 207896 "" "Prometheus/2.33.4" 2024-04-22T21:16:21.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:21 smithi130 ceph-mon[21867]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:21.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:21 smithi130 ceph-mon[26394]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:21.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:21 smithi150 ceph-mon[22676]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:23.509 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:23 smithi130 ceph-mon[21867]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:23.509 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:23 smithi130 ceph-mon[26394]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:23.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:23 smithi150 ceph-mon[22676]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:23.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:16:23 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:16:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:16:23.803 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:16:23 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:16:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:16:25.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:25 smithi130 ceph-mon[21867]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:25.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:25 smithi130 ceph-mon[26394]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:25.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:25 smithi150 ceph-mon[22676]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:26.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:26 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:26.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:26 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:26.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:26 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:26.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:16:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:16:27.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:27 smithi150 ceph-mon[22676]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:27.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:27 smithi130 ceph-mon[21867]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:27.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:27 smithi130 ceph-mon[26394]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:29.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:29 smithi130 ceph-mon[21867]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:29.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:29 smithi130 ceph-mon[26394]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:29.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:29 smithi150 ceph-mon[22676]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:30.303 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:16:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:16:29] "GET /metrics HTTP/1.1" 200 207894 "" "Prometheus/2.33.4" 2024-04-22T21:16:31.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:31 smithi130 ceph-mon[21867]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:31.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:31 smithi130 ceph-mon[26394]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:31.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:31 smithi150 ceph-mon[22676]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:33.495 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:33 smithi130 ceph-mon[21867]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:33.495 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:33 smithi130 ceph-mon[26394]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:33.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:33 smithi150 ceph-mon[22676]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:33.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:16:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:16:33.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:16:33.803 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:16:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:16:33.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:16:33.803 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:16:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:16:33.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:16:35.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:35 smithi130 ceph-mon[21867]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:35.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:35 smithi130 ceph-mon[26394]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:35.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:35 smithi150 ceph-mon[22676]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:36.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:36 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:36.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:36 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:36.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:36 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:36.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:16:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:16:37.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:37 smithi150 ceph-mon[22676]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:37.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:37 smithi130 ceph-mon[21867]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:37.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:37 smithi130 ceph-mon[26394]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:39.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:39 smithi130 ceph-mon[21867]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:39.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:39 smithi130 ceph-mon[26394]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:39.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:39 smithi150 ceph-mon[22676]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:40.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:16:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:16:39] "GET /metrics HTTP/1.1" 200 207894 "" "Prometheus/2.33.4" 2024-04-22T21:16:41.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:41 smithi130 ceph-mon[21867]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:41.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:41 smithi130 ceph-mon[26394]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:41.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:41 smithi150 ceph-mon[22676]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:43.495 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:43 smithi130 ceph-mon[21867]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:43.496 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:43 smithi130 ceph-mon[26394]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:43.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:43 smithi150 ceph-mon[22676]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:43.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:16:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:16:43.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:16:43.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:16:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:16:43.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:16:43.803 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:16:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:16:43.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:16:45.302 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:45 smithi130 ceph-mon[26394]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:45.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:45 smithi130 ceph-mon[21867]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:45.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:45 smithi150 ceph-mon[22676]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:46.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:46 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:46.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:46 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:46.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:46 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:46.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:46 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:16:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:16:47.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:47 smithi150 ceph-mon[22676]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:47.454 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:47 smithi130 ceph-mon[26394]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:47.454 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:47 smithi130 ceph-mon[21867]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:49.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:49 smithi130 ceph-mon[21867]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:49.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:49 smithi130 ceph-mon[26394]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:49.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:49 smithi150 ceph-mon[22676]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:50.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:16:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:16:49] "GET /metrics HTTP/1.1" 200 207887 "" "Prometheus/2.33.4" 2024-04-22T21:16:51.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:51 smithi130 ceph-mon[21867]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:51.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:51 smithi130 ceph-mon[26394]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:51.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:51 smithi150 ceph-mon[22676]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:53.496 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:53 smithi130 ceph-mon[21867]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:53.496 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:53 smithi130 ceph-mon[26394]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:53.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:53 smithi150 ceph-mon[22676]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:53.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:16:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:16:53.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:16:53.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:16:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:16:53.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:16:53.803 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:16:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:16:53.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:16:55.302 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:55 smithi130 ceph-mon[26394]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:55.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:55 smithi130 ceph-mon[21867]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:55.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:55 smithi150 ceph-mon[22676]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:56.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:56 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:56.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:56 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:56 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:16:56.856 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:16:56 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:16:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:16:57.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:57 smithi150 ceph-mon[22676]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:57.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:57 smithi130 ceph-mon[26394]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:57 smithi130 ceph-mon[21867]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:16:59.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:16:59 smithi130 ceph-mon[26394]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:59.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:16:59 smithi130 ceph-mon[21867]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:16:59.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:16:59 smithi150 ceph-mon[22676]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:00.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:16:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:16:59] "GET /metrics HTTP/1.1" 200 207895 "" "Prometheus/2.33.4" 2024-04-22T21:17:01.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:01 smithi130 ceph-mon[21867]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:01.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:01 smithi130 ceph-mon[26394]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:01.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:01 smithi150 ceph-mon[22676]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:03.496 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:03 smithi130 ceph-mon[21867]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:03.496 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:03 smithi130 ceph-mon[26394]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:03.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:03 smithi150 ceph-mon[22676]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:03.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:17:03.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:17:03.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:17:03.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:17:03.803 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:17:03.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:17:05.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:05 smithi150 ceph-mon[22676]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:05.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:05 smithi130 ceph-mon[21867]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:05 smithi130 ceph-mon[26394]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:06.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:06 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:06.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:06 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:06.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:06 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:06.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:17:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:17:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:17:07.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:07 smithi150 ceph-mon[22676]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:07.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:07 smithi130 ceph-mon[21867]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:07.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:07 smithi130 ceph-mon[26394]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:09.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:09 smithi130 ceph-mon[21867]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:09.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:09 smithi130 ceph-mon[26394]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:09.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:09 smithi150 ceph-mon[22676]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:17:09 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:17:09] "GET /metrics HTTP/1.1" 200 207895 "" "Prometheus/2.33.4" 2024-04-22T21:17:10.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:17:10.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:17:10.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:17:10.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:17:10.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:17:10.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:17:11.507 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:11 smithi150 ceph-mon[22676]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:11.507 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:17:11.507 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:11 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:17:11.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:11 smithi130 ceph-mon[21867]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:11.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:17:11.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:11 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:17:11.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:11 smithi130 ceph-mon[26394]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:11.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:17:11.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:11 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:17:13.498 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:13 smithi130 ceph-mon[21867]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:13.498 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:13 smithi130 ceph-mon[26394]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:13.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:13 smithi150 ceph-mon[22676]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:13.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:17:13.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:17:13.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:17:13.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:17:13.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:17:13.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:17:15.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:15 smithi150 ceph-mon[22676]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:15.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:15 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:17:15.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:15 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:17:15.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:15 smithi130 ceph-mon[26394]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:15.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:15 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:17:15.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:15 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:17:15.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:15 smithi130 ceph-mon[21867]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:15.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:15 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:17:15.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:15 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:17:16.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:16 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:16.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:16 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:16 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:16.856 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:17:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:17:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:17:17.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:17 smithi150 ceph-mon[22676]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:17.453 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:17 smithi130 ceph-mon[21867]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:17.454 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:17 smithi130 ceph-mon[26394]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:19.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:19 smithi130 ceph-mon[26394]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:19.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:19 smithi130 ceph-mon[21867]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:19.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:19 smithi150 ceph-mon[22676]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:20.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:17:19 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:17:19] "GET /metrics HTTP/1.1" 200 207895 "" "Prometheus/2.33.4" 2024-04-22T21:17:21.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:21 smithi130 ceph-mon[21867]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:21.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:21 smithi130 ceph-mon[26394]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:21.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:21 smithi150 ceph-mon[22676]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:23.498 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:23 smithi130 ceph-mon[21867]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:23.498 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:23 smithi130 ceph-mon[26394]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:23.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:23 smithi150 ceph-mon[22676]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:23.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:23 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:17:23.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:17:23.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:23 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:17:23.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:17:23.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:23 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:17:23.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:17:25.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:25 smithi150 ceph-mon[22676]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:25.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:25 smithi130 ceph-mon[26394]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:25.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:25 smithi130 ceph-mon[21867]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:26.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:26 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:26.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:26 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:26.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:26 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:26.605 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:17:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:17:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:17:27.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:27 smithi150 ceph-mon[22676]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:27.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:27 smithi130 ceph-mon[21867]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:27.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:27 smithi130 ceph-mon[26394]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:29.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:29 smithi130 ceph-mon[21867]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:29.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:29 smithi130 ceph-mon[26394]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:29.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:29 smithi150 ceph-mon[22676]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:30.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:17:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:17:29] "GET /metrics HTTP/1.1" 200 207891 "" "Prometheus/2.33.4" 2024-04-22T21:17:31.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:31 smithi130 ceph-mon[26394]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:31.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:31 smithi130 ceph-mon[21867]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:31.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:31 smithi150 ceph-mon[22676]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:33.499 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:33 smithi130 ceph-mon[21867]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:33.499 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:33 smithi130 ceph-mon[26394]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:33.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:33 smithi150 ceph-mon[22676]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:33.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:17:33.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:17:33.803 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:17:33.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:17:33.803 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:17:33.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:17:35.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:35 smithi150 ceph-mon[22676]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:35.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:35 smithi130 ceph-mon[26394]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:35.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:35 smithi130 ceph-mon[21867]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:36.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:36 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:36.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:36 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:36.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:36 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:36.605 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:17:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:17:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:17:37.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:37 smithi150 ceph-mon[22676]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:37.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:37 smithi130 ceph-mon[21867]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:37.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:37 smithi130 ceph-mon[26394]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:39.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:39 smithi130 ceph-mon[21867]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:39.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:39 smithi130 ceph-mon[26394]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:39.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:39 smithi150 ceph-mon[22676]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:40.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:17:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:17:39] "GET /metrics HTTP/1.1" 200 207891 "" "Prometheus/2.33.4" 2024-04-22T21:17:41.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:41 smithi130 ceph-mon[21867]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:41.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:41 smithi130 ceph-mon[26394]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:41.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:41 smithi150 ceph-mon[22676]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:43.499 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:43 smithi130 ceph-mon[26394]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:43.500 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:43 smithi130 ceph-mon[21867]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:43.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:43 smithi150 ceph-mon[22676]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:43.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:17:43.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:17:43.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:17:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:17:43.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:17:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:17:45.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:45 smithi150 ceph-mon[22676]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:45.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:45 smithi130 ceph-mon[26394]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:45 smithi130 ceph-mon[21867]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:46.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:46 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:46.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:46 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:46.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:46 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:46.855 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:17:46 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:17:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:17:47.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:47 smithi150 ceph-mon[22676]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:47.453 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:47 smithi130 ceph-mon[26394]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:47.453 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:47 smithi130 ceph-mon[21867]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:49.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:49 smithi130 ceph-mon[26394]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:49.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:49 smithi130 ceph-mon[21867]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:49.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:49 smithi150 ceph-mon[22676]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:50.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:17:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:17:49] "GET /metrics HTTP/1.1" 200 207896 "" "Prometheus/2.33.4" 2024-04-22T21:17:51.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:51 smithi130 ceph-mon[26394]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:51.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:51 smithi130 ceph-mon[21867]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:51.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:51 smithi150 ceph-mon[22676]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:53.500 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:53 smithi130 ceph-mon[21867]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:53.500 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:53 smithi130 ceph-mon[26394]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:53.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:53 smithi150 ceph-mon[22676]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:53.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:17:53.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:17:53.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:17:53.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:17:53.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:17:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:17:53.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:17:55.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:55 smithi150 ceph-mon[22676]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:55.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:55 smithi130 ceph-mon[26394]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:55.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:55 smithi130 ceph-mon[21867]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:56.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:56 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:56.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:56 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:56.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:56 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:17:56.855 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:17:56 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:17:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:17:57.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:57 smithi150 ceph-mon[22676]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:57.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:57 smithi130 ceph-mon[26394]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:57 smithi130 ceph-mon[21867]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:17:59.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:17:59 smithi130 ceph-mon[26394]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:59.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:17:59 smithi130 ceph-mon[21867]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:17:59.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:17:59 smithi150 ceph-mon[22676]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:00.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:17:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:17:59] "GET /metrics HTTP/1.1" 200 207893 "" "Prometheus/2.33.4" 2024-04-22T21:18:01.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:01 smithi130 ceph-mon[21867]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:01.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:01 smithi130 ceph-mon[26394]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:01.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:01 smithi150 ceph-mon[22676]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:03.500 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:03 smithi130 ceph-mon[26394]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:03.501 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:03 smithi130 ceph-mon[21867]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:03.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:03 smithi150 ceph-mon[22676]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:03.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:18:03.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:18:03.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:18:03.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:18:03.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:18:03.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:18:05.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:05 smithi150 ceph-mon[22676]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:05.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:05 smithi130 ceph-mon[26394]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:05.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:05 smithi130 ceph-mon[21867]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:06.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:06 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:06.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:06 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:06.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:06 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:06.605 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:18:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:18:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:18:07.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:07 smithi150 ceph-mon[22676]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:07.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:07 smithi130 ceph-mon[26394]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:07.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:07 smithi130 ceph-mon[21867]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:09.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:09 smithi130 ceph-mon[26394]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:09.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:09 smithi130 ceph-mon[21867]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:09.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:09 smithi150 ceph-mon[22676]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:10.188 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:09 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:18:09] "GET /metrics HTTP/1.1" 200 207893 "" "Prometheus/2.33.4" 2024-04-22T21:18:10.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:18:10.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:10 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:18:10.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:18:10.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:10 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:18:10.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:18:10.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:10 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:18:11.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:11 smithi130 ceph-mon[21867]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:11.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:11 smithi130 ceph-mon[26394]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:11.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:11 smithi150 ceph-mon[22676]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:13.501 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:13 smithi130 ceph-mon[21867]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:13.502 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:13 smithi130 ceph-mon[26394]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:13.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:13 smithi150 ceph-mon[22676]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:13.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:18:13.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:18:13.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:18:13.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:18:13.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:18:13.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:18:15.327 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:15 smithi130 ceph-mon[26394]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:15.327 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:15 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:18:15.327 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:15 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:18:15.327 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:15 smithi130 ceph-mon[21867]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:15.328 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:15 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:18:15.328 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:15 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:18:15.330 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:15 smithi150 ceph-mon[22676]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:15.330 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:15 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:18:15.330 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:15 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:18:16.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:16 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:16.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:16 smithi150 ceph-mon[22676]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:18:16.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:16 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:16.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:16 smithi130 ceph-mon[21867]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:18:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:16 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:16 smithi130 ceph-mon[26394]: from='mgr.14445 172.21.15.130:0/3725667686' entity='mgr.y' 2024-04-22T21:18:16.855 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:18:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:18:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:18:17.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:17 smithi150 ceph-mon[22676]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:17.453 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:17 smithi130 ceph-mon[21867]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:17.453 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:17 smithi130 ceph-mon[26394]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:19.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:19 smithi130 ceph-mon[21867]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:19.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:19 smithi130 ceph-mon[26394]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:19.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:19 smithi150 ceph-mon[22676]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:20.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:19 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:18:19] "GET /metrics HTTP/1.1" 200 207891 "" "Prometheus/2.33.4" 2024-04-22T21:18:21.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:21 smithi130 ceph-mon[21867]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:21.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:21 smithi130 ceph-mon[26394]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:21.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:21 smithi150 ceph-mon[22676]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:23.502 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:23 smithi130 ceph-mon[26394]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:23.503 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:23 smithi130 ceph-mon[21867]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:23.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:23 smithi150 ceph-mon[22676]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:23.801 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:23 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:18:23.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:18:23.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:23 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:18:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:18:23.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:23 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:18:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:18:25.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:25 smithi150 ceph-mon[22676]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:25.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:25 smithi130 ceph-mon[26394]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:25.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:25 smithi130 ceph-mon[21867]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:26.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:26 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:26.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:26 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:26.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:26 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:26.855 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:18:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:18:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:18:27.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:27 smithi150 ceph-mon[22676]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:27.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:27 smithi130 ceph-mon[26394]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:27.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:27 smithi130 ceph-mon[21867]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:29.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:29 smithi130 ceph-mon[26394]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:29.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:29 smithi130 ceph-mon[21867]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:29.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:29 smithi150 ceph-mon[22676]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:30.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:18:29] "GET /metrics HTTP/1.1" 200 207896 "" "Prometheus/2.33.4" 2024-04-22T21:18:31.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:31 smithi130 ceph-mon[21867]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:31.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:31 smithi130 ceph-mon[26394]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:31.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:31 smithi150 ceph-mon[22676]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:33.503 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:33 smithi130 ceph-mon[26394]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:33.503 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:33 smithi130 ceph-mon[21867]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:33.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:33 smithi150 ceph-mon[22676]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:33.801 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:18:33.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:18:33.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:18:33.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:18:33.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:18:33.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:18:35.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:35 smithi150 ceph-mon[22676]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:35.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:35 smithi130 ceph-mon[26394]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:35.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:35 smithi130 ceph-mon[21867]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:36.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:36 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:36.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:36 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:36.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:36 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:36.605 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:18:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:18:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:18:37.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:37 smithi150 ceph-mon[22676]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:37.515 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:37 smithi130 ceph-mon[21867]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:37.516 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:37 smithi130 ceph-mon[26394]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:37.545 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps' 2024-04-22T21:18:39.546 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:18:39.546 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (6m) 2m ago 7m 24.2M - ba2b418f427c 2176cd6f1d6a 2024-04-22T21:18:39.546 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (7m) 84s ago 7m 55.6M - 8.3.5 dad864ee21e9 b3f3edd15758 2024-04-22T21:18:39.546 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (6m) 2m ago 6m 51.3M - 3.5 e1d6a67b021e 264d4f175cde 2024-04-22T21:18:39.546 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283 running (2m) 84s ago 12m 450M - 19.0.0-3198-g56e81df3 82086b19bb07 327605fe868f 2024-04-22T21:18:39.546 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:9283 running (13m) 2m ago 13m 460M - 17.2.0 e1d6a67b021e 878d76986ae8 2024-04-22T21:18:39.547 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (13m) 2m ago 13m 49.8M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:18:39.547 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (12m) 84s ago 12m 40.4M 2048M 17.2.0 e1d6a67b021e 861909a68cf9 2024-04-22T21:18:39.547 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (12m) 2m ago 12m 40.2M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:18:39.547 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (8m) 2m ago 8m 14.5M - 1dbe0e931976 82b1ae40160e 2024-04-22T21:18:39.547 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (8m) 84s ago 8m 16.8M - 1dbe0e931976 aafb83793157 2024-04-22T21:18:39.547 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (11m) 2m ago 11m 50.4M 4016M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:18:39.547 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (11m) 2m ago 11m 52.5M 4016M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:18:39.547 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (10m) 2m ago 10m 45.7M 4016M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:18:39.547 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (10m) 2m ago 10m 48.6M 4016M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:18:39.547 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (10m) 84s ago 10m 49.5M 4272M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:18:39.547 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (9m) 84s ago 9m 48.3M 4272M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:18:39.548 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (9m) 84s ago 9m 49.0M 4272M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:18:39.548 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (8m) 84s ago 8m 49.2M 4272M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:18:39.548 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (6m) 84s ago 8m 64.3M - 514e6a882f6e afc570412b78 2024-04-22T21:18:39.548 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (6m) 2m ago 6m 71.9M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:18:39.548 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (6m) 84s ago 6m 70.1M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:18:39.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:39 smithi130 ceph-mon[21867]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:39.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:39 smithi130 ceph-mon[26394]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:39.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:39 smithi150 ceph-mon[22676]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:40.229 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:18:39] "GET /metrics HTTP/1.1" 200 207896 "" "Prometheus/2.33.4" 2024-04-22T21:18:40.253 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions' 2024-04-22T21:18:41.442 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:41 smithi130 ceph-mon[21867]: from='client.24916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:18:41.442 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:41 smithi130 ceph-mon[21867]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:41 smithi130 ceph-mon[26394]: from='client.24916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:18:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:41 smithi130 ceph-mon[26394]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:41.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:41 smithi150 ceph-mon[22676]: from='client.24916 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:18:41.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:41 smithi150 ceph-mon[22676]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:42.314 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:18:42.315 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:18:42.315 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T21:18:42.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:18:42.315 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:18:42.315 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T21:18:42.315 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 1 2024-04-22T21:18:42.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:18:42.315 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:18:42.315 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:18:42.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:18:42.316 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": {}, 2024-04-22T21:18:42.316 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:18:42.316 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:18:42.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:18:42.316 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:18:42.316 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-04-22T21:18:42.316 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 1 2024-04-22T21:18:42.316 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:18:42.316 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:18:42.927 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph -s' 2024-04-22T21:18:43.464 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:43 smithi130 ceph-mon[26394]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:43.464 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:43 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3556886712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:18:43.464 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:43 smithi130 ceph-mon[21867]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:43.464 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:43 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3556886712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:18:43.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:43 smithi150 ceph-mon[22676]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:43.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:43 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3556886712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:18:43.801 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:18:43.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:18:43.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:18:43.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:18:43.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:18:43.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:18:44.929 INFO:teuthology.orchestra.run.smithi130.stdout: cluster: 2024-04-22T21:18:44.929 INFO:teuthology.orchestra.run.smithi130.stdout: id: dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:18:44.929 INFO:teuthology.orchestra.run.smithi130.stdout: health: HEALTH_OK 2024-04-22T21:18:44.929 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:18:44.929 INFO:teuthology.orchestra.run.smithi130.stdout: services: 2024-04-22T21:18:44.929 INFO:teuthology.orchestra.run.smithi130.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-04-22T21:18:44.929 INFO:teuthology.orchestra.run.smithi130.stdout: mgr: y(active, since 8m), standbys: x 2024-04-22T21:18:44.930 INFO:teuthology.orchestra.run.smithi130.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-04-22T21:18:44.930 INFO:teuthology.orchestra.run.smithi130.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-22T21:18:44.930 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:18:44.930 INFO:teuthology.orchestra.run.smithi130.stdout: data: 2024-04-22T21:18:44.930 INFO:teuthology.orchestra.run.smithi130.stdout: pools: 6 pools, 161 pgs 2024-04-22T21:18:44.930 INFO:teuthology.orchestra.run.smithi130.stdout: objects: 209 objects, 585 KiB 2024-04-22T21:18:44.930 INFO:teuthology.orchestra.run.smithi130.stdout: usage: 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:18:44.930 INFO:teuthology.orchestra.run.smithi130.stdout: pgs: 161 active+clean 2024-04-22T21:18:44.930 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:18:44.930 INFO:teuthology.orchestra.run.smithi130.stdout: io: 2024-04-22T21:18:44.930 INFO:teuthology.orchestra.run.smithi130.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-04-22T21:18:44.930 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:18:45.234 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:45 smithi130 ceph-mon[21867]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:45.234 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:45 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4125718777' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:18:45.234 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:45 smithi130 ceph-mon[26394]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:45.234 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:45 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4125718777' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:18:45.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:45 smithi150 ceph-mon[22676]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:45.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:45 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4125718777' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:18:45.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph health detail' 2024-04-22T21:18:46.229 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:46 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:46.230 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:46 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:46.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:46 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:46.605 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:18:46 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:18:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:18:47.197 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:47 smithi130 ceph-mon[26394]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:47.197 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:47 smithi130 ceph-mon[21867]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:47.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:47 smithi150 ceph-mon[22676]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:47.538 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-04-22T21:18:48.171 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-04-22T21:18:48.301 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:48 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3950933721' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:18:48.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:48 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3950933721' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:18:48.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:48 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3950933721' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:18:49.381 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:49 smithi130 ceph-mon[21867]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:49.381 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:49 smithi130 ceph-mon[26394]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:49.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:49 smithi150 ceph-mon[22676]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:50.051 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:18:49] "GET /metrics HTTP/1.1" 200 207892 "" "Prometheus/2.33.4" 2024-04-22T21:18:50.081 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:18:50.404 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:50 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/340938967' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:18:50.404 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:50 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/340938967' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:18:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:50 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/340938967' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:18:50.704 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph mgr fail' 2024-04-22T21:18:51.301 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:51 smithi130 ceph-mon[26394]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:51.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:51 smithi130 ceph-mon[21867]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:51.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:51 smithi150 ceph-mon[22676]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:18:53.502 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-mon[21867]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:53.503 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/217540817' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T21:18:53.503 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-mon[21867]: osdmap e96: 8 total, 8 up, 8 in 2024-04-22T21:18:53.503 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:53.208+0000 7f4a654ac700 -1 mgr handle_mgr_map I was active but no longer amignoring --setuser ceph since I am not root 2024-04-22T21:18:53.503 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ignoring --setgroup ceph since I am not root 2024-04-22T21:18:53.503 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-mgr[22097]: -- 172.21.15.130:0/3926818552 <== mon.1 v2:172.21.15.130:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562afc758340 con 0x562afc730800 2024-04-22T21:18:53.503 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:53.378+0000 7f07e777f000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:18:53.504 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-mon[26394]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:53.504 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/217540817' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T21:18:53.504 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-mon[26394]: osdmap e96: 8 total, 8 up, 8 in 2024-04-22T21:18:53.516 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:53 smithi150 ceph-mon[22676]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:18:53.516 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:53 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/217540817' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T21:18:53.516 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:53 smithi150 ceph-mon[22676]: osdmap e96: 8 total, 8 up, 8 in 2024-04-22T21:18:53.516 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:18:53 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:18:53] ENGINE Bus STOPPING 2024-04-22T21:18:53.517 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:18:53 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:18:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T21:18:53.517 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:18:53 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:18:53] ENGINE Bus STOPPED 2024-04-22T21:18:53.517 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:18:53 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:18:53] ENGINE Bus STARTING 2024-04-22T21:18:53.517 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:18:53 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:18:53] ENGINE Serving on http://:::9283 2024-04-22T21:18:53.517 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:18:53 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:18:53] ENGINE Bus STARTED 2024-04-22T21:18:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:53.514+0000 7f07e777f000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:18:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:53.617+0000 7f07e777f000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:18:53.801 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:18:53.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:18:53.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:18:53.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2024-04-22T21:18:53.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:18:53.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-22T21:18:53.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'sleep 180' 2024-04-22T21:18:54.201 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:18:53.995Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=2 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:18:54.201 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:53.945+0000 7f07e777f000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:18:54.201 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:54.033+0000 7f07e777f000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:18:54.517 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/217540817' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T21:18:54.517 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: mgrmap e22: x(active, starting, since 0.525226s) 2024-04-22T21:18:54.517 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:18:54.517 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:18:54.517 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:18:54.517 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:18:54.517 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:18:54.517 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:18:54.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:18:54.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:18:54.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:18:54.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:18:54.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:18:54.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:18:54.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:18:54.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:18:54.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:18:54.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: Manager daemon x is now available 2024-04-22T21:18:54.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:18:54.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: Queued rgw.foo for migration 2024-04-22T21:18:54.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-22T21:18:54.519 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:18:54.519 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:18:54.519 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:18:54.519 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:18:54.519 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T21:18:54.519 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T21:18:54.519 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: Deploying cephadm binary to smithi130 2024-04-22T21:18:54.519 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: Deploying cephadm binary to smithi150 2024-04-22T21:18:54.519 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:18:54.519 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:18:54.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/217540817' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T21:18:54.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: mgrmap e22: x(active, starting, since 0.525226s) 2024-04-22T21:18:54.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:18:54.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:18:54.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:18:54.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:18:54.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:18:54.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:18:54.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:18:54.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:18:54.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:18:54.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: Manager daemon x is now available 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: Queued rgw.foo for migration 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:18:54.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T21:18:54.522 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T21:18:54.522 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: Deploying cephadm binary to smithi130 2024-04-22T21:18:54.522 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: Deploying cephadm binary to smithi150 2024-04-22T21:18:54.522 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:18:54.522 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:54 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:18:54.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/217540817' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T21:18:54.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: mgrmap e22: x(active, starting, since 0.525226s) 2024-04-22T21:18:54.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:18:54.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:18:54.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:18:54.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:18:54.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:18:54.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:18:54.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:18:54.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:18:54.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:18:54.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:18:54.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:18:54.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: Manager daemon x is now available 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: Queued rgw.foo for migration 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T21:18:54.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: Deploying cephadm binary to smithi130 2024-04-22T21:18:54.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: Deploying cephadm binary to smithi150 2024-04-22T21:18:54.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:18:54.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:54 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:18:55.295 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:55.195+0000 7f07e777f000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:18:55.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:55 smithi130 ceph-mon[26394]: mgrmap e23: x(active, since 1.53691s) 2024-04-22T21:18:55.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:55 smithi130 ceph-mon[26394]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:18:55.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:55 smithi130 ceph-mon[26394]: [22/Apr/2024:21:18:54] ENGINE Bus STARTING 2024-04-22T21:18:55.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:55 smithi130 ceph-mon[21867]: mgrmap e23: x(active, since 1.53691s) 2024-04-22T21:18:55.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:55 smithi130 ceph-mon[21867]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:18:55.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:55 smithi130 ceph-mon[21867]: [22/Apr/2024:21:18:54] ENGINE Bus STARTING 2024-04-22T21:18:55.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:55.370+0000 7f07e777f000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:18:55.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:55.454+0000 7f07e777f000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:18:55.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:55 smithi150 ceph-mon[22676]: mgrmap e23: x(active, since 1.53691s) 2024-04-22T21:18:55.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:55 smithi150 ceph-mon[22676]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:18:55.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:55 smithi150 ceph-mon[22676]: [22/Apr/2024:21:18:54] ENGINE Bus STARTING 2024-04-22T21:18:55.955 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:55.781+0000 7f07e777f000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:18:55.955 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:55.873+0000 7f07e777f000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:18:56.210 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:55.966+0000 7f07e777f000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:18:56.211 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:56.209+0000 7f07e777f000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:18:56.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-mon[21867]: [22/Apr/2024:21:18:55] ENGINE Serving on https://172.21.15.150:7150 2024-04-22T21:18:56.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:56.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-mon[21867]: [22/Apr/2024:21:18:55] ENGINE Serving on http://172.21.15.150:8765 2024-04-22T21:18:56.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-mon[21867]: [22/Apr/2024:21:18:55] ENGINE Bus STARTED 2024-04-22T21:18:56.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-mon[21867]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:18:56.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-mon[21867]: mgrmap e24: x(active, since 2s) 2024-04-22T21:18:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-mon[26394]: [22/Apr/2024:21:18:55] ENGINE Serving on https://172.21.15.150:7150 2024-04-22T21:18:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-mon[26394]: [22/Apr/2024:21:18:55] ENGINE Serving on http://172.21.15.150:8765 2024-04-22T21:18:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-mon[26394]: [22/Apr/2024:21:18:55] ENGINE Bus STARTED 2024-04-22T21:18:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-mon[26394]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:18:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-mon[26394]: mgrmap e24: x(active, since 2s) 2024-04-22T21:18:56.605 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:18:56 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:18:56] "GET /metrics HTTP/1.1" 200 34908 "" "Prometheus/2.33.4" 2024-04-22T21:18:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:56 smithi150 ceph-mon[22676]: [22/Apr/2024:21:18:55] ENGINE Serving on https://172.21.15.150:7150 2024-04-22T21:18:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:56 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:18:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:56 smithi150 ceph-mon[22676]: [22/Apr/2024:21:18:55] ENGINE Serving on http://172.21.15.150:8765 2024-04-22T21:18:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:56 smithi150 ceph-mon[22676]: [22/Apr/2024:21:18:55] ENGINE Bus STARTED 2024-04-22T21:18:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:56 smithi150 ceph-mon[22676]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:18:56.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:56 smithi150 ceph-mon[22676]: mgrmap e24: x(active, since 2s) 2024-04-22T21:18:57.051 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:56 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:56.651+0000 7f07e777f000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:18:57.516 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:57 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:57.138+0000 7f07e777f000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:18:57.516 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:57 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:57.233+0000 7f07e777f000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:18:57.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:57 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:57.528+0000 7f07e777f000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:18:57.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:57 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:57.619+0000 7f07e777f000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:18:58.301 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:58 smithi130 ceph-mon[26394]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:18:58.301 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:18:58 smithi130 ceph-mon[26394]: mgrmap e25: x(active, since 4s) 2024-04-22T21:18:58.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:58.023+0000 7f07e777f000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:18:58.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:58.156+0000 7f07e777f000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:18:58.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:58.259+0000 7f07e777f000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:18:58.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:58 smithi130 ceph-mon[21867]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:18:58.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:18:58 smithi130 ceph-mon[21867]: mgrmap e25: x(active, since 4s) 2024-04-22T21:18:58.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:58 smithi150 ceph-mon[22676]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:18:58.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:18:58 smithi150 ceph-mon[22676]: mgrmap e25: x(active, since 4s) 2024-04-22T21:18:58.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:58.378+0000 7f07e777f000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:18:58.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:58.469+0000 7f07e777f000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:18:59.105 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:18:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:18:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: |\n OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked down and back up at {{ $value | humanize }} times once a minute for 5 minutes. This could indicate a network issue (latency, packet drop, disruption) on the clusters \"cluster network\". Check the network environment on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSD's to flap (mark each other out)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.130:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:18:59.105 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:18:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:18:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: |\n OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by more than 30% from average PG count.\n summary: PG allocations are not balanced across devices\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.130:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:18:59.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:59.055+0000 7f07e777f000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:18:59.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:59.440+0000 7f07e777f000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:18:59.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:18:59] ENGINE Bus STARTING 2024-04-22T21:18:59.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: CherryPy Checker: 2024-04-22T21:18:59.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: The Application mounted at '' has an empty config. 2024-04-22T21:18:59.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: 2024-04-22T21:18:59.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:18:59] ENGINE Serving on http://:::9283 2024-04-22T21:18:59.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: [22/Apr/2024:21:18:59] ENGINE Bus STARTED 2024-04-22T21:19:00.263 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:18:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:18:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:19:00.263 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:00 smithi130 ceph-mon[26394]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:19:00.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:00 smithi130 ceph-mon[26394]: Standby manager daemon y started 2024-04-22T21:19:00.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:00 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.130:0/2184194644' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T21:19:00.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:00 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.130:0/2184194644' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:19:00.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:00 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.130:0/2184194644' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T21:19:00.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:00 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.130:0/2184194644' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:19:00.264 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:00 smithi130 ceph-mon[21867]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:19:00.264 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:00 smithi130 ceph-mon[21867]: Standby manager daemon y started 2024-04-22T21:19:00.264 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:00 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.130:0/2184194644' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T21:19:00.264 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:00 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.130:0/2184194644' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:19:00.264 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:00 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.130:0/2184194644' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T21:19:00.265 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:00 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.130:0/2184194644' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:19:00.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:00 smithi150 ceph-mon[22676]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:19:00.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:00 smithi150 ceph-mon[22676]: Standby manager daemon y started 2024-04-22T21:19:00.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:00 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.130:0/2184194644' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T21:19:00.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:00 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.130:0/2184194644' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:19:00.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:00 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.130:0/2184194644' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T21:19:00.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:00 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.130:0/2184194644' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:19:00.539 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:00 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:19:00.276Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:19:01.178 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:01 smithi150 ceph-mon[22676]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-22T21:19:01.178 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:01 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:19:01.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:01 smithi130 ceph-mon[21867]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-22T21:19:01.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:01 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:19:01.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:01 smithi130 ceph-mon[26394]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-22T21:19:01.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:01 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:19:02.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:02 smithi150 ceph-mon[22676]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:19:02.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:02 smithi130 ceph-mon[26394]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:19:02.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:02 smithi130 ceph-mon[21867]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:19:03.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:03 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:03.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:03 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:03.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:03 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:03.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:03 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:03.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:03 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:03.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:03 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:03.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:03 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:03.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:03 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:03.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:03 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:03.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:03 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:03.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:03 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:03.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:03 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:03.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=error ts=2024-04-22T21:19:03.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:19:03.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:19:03.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.130:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.130 because it doesn't contain any IP SANs" 2024-04-22T21:19:03.802 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=warn ts=2024-04-22T21:19:03.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.150:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.150 because it doesn't contain any IP SANs" 2024-04-22T21:19:04.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:19:04.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: Adjusting osd_memory_target on smithi130 to 2992M 2024-04-22T21:19:04.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:19:04.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:19:04.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:19:04.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: Adjusting osd_memory_target on smithi130 to 2992M 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:19:04.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:04 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:19:04.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:19:04.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: Adjusting osd_memory_target on smithi130 to 2992M 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T21:19:04.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:04.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:19:04.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:04 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:19:05.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: Adjusting osd_memory_target on smithi150 to 3248M 2024-04-22T21:19:05.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.553 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: Adjusting osd_memory_target on smithi150 to 3248M 2024-04-22T21:19:05.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:19:05.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:19:05.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:19:05.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:19:05.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:19:05.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:19:05.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:19:05.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:19:05.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:05 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: Adjusting osd_memory_target on smithi150 to 3248M 2024-04-22T21:19:05.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:19:05.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:05 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:05.979 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:05 smithi130 systemd[1]: Stopping Ceph node-exporter.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:19:05.979 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:05 smithi130 bash[55516]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter.a" found: no such container 2024-04-22T21:19:05.979 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:05 smithi130 podman[55529]: 2024-04-22 21:19:05.776441423 +0000 UTC m=+0.135004104 container died 82b1ae40160ec66f6730ad62a2dc36e162e31717a94ab12e6024e2f8dcaf5cb8 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:06.271 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 podman[55529]: 2024-04-22 21:19:06.090830259 +0000 UTC m=+0.449392930 container cleanup 82b1ae40160ec66f6730ad62a2dc36e162e31717a94ab12e6024e2f8dcaf5cb8 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:06.271 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 bash[55529]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a 2024-04-22T21:19:06.538 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-mon[21867]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-22T21:19:06.539 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-mon[21867]: Reconfiguring daemon node-exporter.a on smithi130 2024-04-22T21:19:06.539 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-mon[21867]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:19:06.539 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-mon[21867]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:19:06.539 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-mon[26394]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-22T21:19:06.539 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-mon[26394]: Reconfiguring daemon node-exporter.a on smithi130 2024-04-22T21:19:06.539 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-mon[26394]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:19:06.539 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-mon[26394]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:19:06.540 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 podman[55544]: 2024-04-22 21:19:06.283784109 +0000 UTC m=+0.511224235 container remove 82b1ae40160ec66f6730ad62a2dc36e162e31717a94ab12e6024e2f8dcaf5cb8 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:06.540 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-22T21:19:06.540 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 bash[55557]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter.a" found: no such container 2024-04-22T21:19:06.540 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-22T21:19:06.540 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 systemd[1]: Stopped Ceph node-exporter.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:19:06.540 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 3.780s CPU time. 2024-04-22T21:19:06.540 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 systemd[1]: Starting Ceph node-exporter.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:19:06.540 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 podman[55625]: 2024-04-22 21:19:06.43492235 +0000 UTC m=+0.017380525 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-22T21:19:06.540 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 podman[55625]: 2024-04-22 21:19:06.551609736 +0000 UTC m=+0.134067931 container create 4057e105dc28a04c6a5aea80232a5a6e655b5add4669caa59e395fa1696d74ea (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:06.605 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:19:06] "GET /metrics HTTP/1.1" 200 34908 "" "Prometheus/2.33.4" 2024-04-22T21:19:06.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:06 smithi150 ceph-mon[22676]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-22T21:19:06.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:06 smithi150 ceph-mon[22676]: Reconfiguring daemon node-exporter.a on smithi130 2024-04-22T21:19:06.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:06 smithi150 ceph-mon[22676]: from='client.24712 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:19:06.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:06 smithi150 ceph-mon[22676]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:19:06.801 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 podman[55625]: 2024-04-22 21:19:06.770715131 +0000 UTC m=+0.353173292 container init 4057e105dc28a04c6a5aea80232a5a6e655b5add4669caa59e395fa1696d74ea (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:06.801 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 podman[55625]: 2024-04-22 21:19:06.773370864 +0000 UTC m=+0.355829022 container start 4057e105dc28a04c6a5aea80232a5a6e655b5add4669caa59e395fa1696d74ea (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:06.801 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.776Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-22T21:19:06.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.776Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-22T21:19:06.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.776Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-22T21:19:06.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.776Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T21:19:06.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-22T21:19:06.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=arp 2024-04-22T21:19:06.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-22T21:19:06.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-22T21:19:06.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-22T21:19:06.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-22T21:19:06.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-22T21:19:06.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-22T21:19:06.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-22T21:19:06.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-22T21:19:06.803 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=edac 2024-04-22T21:19:06.803 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-22T21:19:06.803 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-22T21:19:06.803 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-22T21:19:06.803 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-22T21:19:06.803 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-22T21:19:06.803 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=os 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-22T21:19:06.804 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=stat 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=time 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=uname 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.777Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-22T21:19:06.805 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[55639]: ts=2024-04-22T21:19:06.778Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-22T21:19:07.301 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 bash[55625]: 4057e105dc28a04c6a5aea80232a5a6e655b5add4669caa59e395fa1696d74ea 2024-04-22T21:19:07.314 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:19:06 smithi130 systemd[1]: Started Ceph node-exporter.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:19:08.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:07 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:08.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:07 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:08.173 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:07 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:08.173 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:07 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:08.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:07 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:08.357 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:07 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:09.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:08 smithi130 ceph-mon[21867]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T21:19:09.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:08 smithi130 ceph-mon[21867]: Reconfiguring daemon alertmanager.a on smithi130 2024-04-22T21:19:09.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:08 smithi130 ceph-mon[21867]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:19:09.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:08 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:19:09.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:08 smithi130 ceph-mon[26394]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T21:19:09.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:08 smithi130 ceph-mon[26394]: Reconfiguring daemon alertmanager.a on smithi130 2024-04-22T21:19:09.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:08 smithi130 ceph-mon[26394]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:19:09.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:08 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:19:09.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:08 smithi150 ceph-mon[22676]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T21:19:09.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:08 smithi150 ceph-mon[22676]: Reconfiguring daemon alertmanager.a on smithi130 2024-04-22T21:19:09.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:08 smithi150 ceph-mon[22676]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:19:09.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:08 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:19:10.225 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:10 smithi130 ceph-mon[21867]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:19:10.225 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:19:09 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[22093]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:19:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:19:10.225 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:10 smithi130 ceph-mon[26394]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:19:10.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:10 smithi150 ceph-mon[22676]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:19:10.551 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:10 smithi130 systemd[1]: Stopping Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:19:10.551 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:10 smithi130 bash[55818]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-22T21:19:10.551 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:10 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[46548]: level=info ts=2024-04-22T21:19:10.432Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-22T21:19:10.552 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:10 smithi130 podman[55831]: 2024-04-22 21:19:10.453284267 +0000 UTC m=+0.122290971 container died 2176cd6f1d6a8bb4ce9befa302e267f9be2e4dbd2c7d4287b9be5383e8375f6c (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:11.051 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:10 smithi130 podman[55831]: 2024-04-22 21:19:10.882482514 +0000 UTC m=+0.551489217 container cleanup 2176cd6f1d6a8bb4ce9befa302e267f9be2e4dbd2c7d4287b9be5383e8375f6c (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:11.051 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:10 smithi130 bash[55831]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a 2024-04-22T21:19:11.464 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 podman[55845]: 2024-04-22 21:19:11.125461266 +0000 UTC m=+0.671561559 container remove 2176cd6f1d6a8bb4ce9befa302e267f9be2e4dbd2c7d4287b9be5383e8375f6c (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:11.464 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 podman[55845]: 2024-04-22 21:19:11.201123371 +0000 UTC m=+0.747223665 volume remove e7b1bf86aa29acd52bcdab65b61cc65c92ec728cba337d813cf627eb4b1fa9ce 2024-04-22T21:19:11.464 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 bash[55857]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-22T21:19:11.465 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-22T21:19:11.465 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 systemd[1]: Stopped Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:19:11.465 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.051s CPU time. 2024-04-22T21:19:11.465 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 systemd[1]: Starting Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:19:11.732 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 podman[55925]: 2024-04-22 21:19:11.477064567 +0000 UTC m=+0.114718588 volume create fe3bb1df2711f659ed6c559592f2bcdc3aafd6bda65efe37147783786bd43e56 2024-04-22T21:19:11.732 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 podman[55925]: 2024-04-22 21:19:11.381006446 +0000 UTC m=+0.018660483 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-22T21:19:11.732 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 podman[55925]: 2024-04-22 21:19:11.560892168 +0000 UTC m=+0.198546190 container create 23fd04f877ce9baea4704909657f4398261f969593a5035cea5f8ef2fcd63505 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:12.051 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 podman[55925]: 2024-04-22 21:19:11.863772037 +0000 UTC m=+0.501426060 container init 23fd04f877ce9baea4704909657f4398261f969593a5035cea5f8ef2fcd63505 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:12.051 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 podman[55925]: 2024-04-22 21:19:11.866421551 +0000 UTC m=+0.504075573 container start 23fd04f877ce9baea4704909657f4398261f969593a5035cea5f8ef2fcd63505 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:12.051 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[55940]: level=info ts=2024-04-22T21:19:11.877Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-22T21:19:12.051 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[55940]: level=info ts=2024-04-22T21:19:11.877Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-22T21:19:12.051 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[55940]: level=info ts=2024-04-22T21:19:11.877Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.130 port=9094 2024-04-22T21:19:12.052 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[55940]: level=info ts=2024-04-22T21:19:11.879Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-22T21:19:12.052 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[55940]: level=info ts=2024-04-22T21:19:11.904Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T21:19:12.052 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[55940]: level=info ts=2024-04-22T21:19:11.904Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T21:19:12.052 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[55940]: level=info ts=2024-04-22T21:19:11.906Z caller=main.go:518 msg=Listening address=:9093 2024-04-22T21:19:12.052 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[55940]: level=info ts=2024-04-22T21:19:11.906Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-22T21:19:12.052 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 bash[55925]: 23fd04f877ce9baea4704909657f4398261f969593a5035cea5f8ef2fcd63505 2024-04-22T21:19:12.052 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:11 smithi130 systemd[1]: Started Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:19:12.451 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:12 smithi130 ceph-mon[26394]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:19:12.451 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:12 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:12.451 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:12 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:12.451 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:12 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:19:12.451 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:12 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:19:12.451 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:12 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:19:12.451 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:12 smithi130 ceph-mon[21867]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:19:12.452 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:12 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:12.452 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:12 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:12.452 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:12 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:19:12.452 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:12 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:19:12.452 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:12 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:19:12.605 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:19:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.150:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.130:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:19:12.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:12 smithi150 ceph-mon[22676]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:19:12.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:12 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:12.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:12 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:12.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:12 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:19:12.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:12 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:19:12.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:12 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:19:13.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:13 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:19:13.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:13 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:19:13.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:13 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:19:13.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:13 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:19:13.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:13 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:19:13.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:13 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:19:14.051 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[55940]: level=info ts=2024-04-22T21:19:13.880Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000749618s 2024-04-22T21:19:14.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:14 smithi130 ceph-mon[26394]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:19:14.436 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:14 smithi130 ceph-mon[21867]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:19:14.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:14 smithi150 ceph-mon[22676]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:19:15.605 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:19:15.222+0000 7f40299f7700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:15.605 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Creating ceph-iscsi config... 2024-04-22T21:19:15.605 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:15.605 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Traceback (most recent call last): 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: return _run_code(code, main_globals, None, 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: exec(code, run_globals) 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:15.606 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:15.607 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:15.607 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:15.607 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:15.607 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Traceback (most recent call last): 2024-04-22T21:19:15.607 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:19:15.607 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:19:15.607 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:19:15.607 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: return future.result(timeout) 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: return self.__get_result() 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: raise self._exception 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: use_current_daemon_image=reconfig, 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:15.608 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Creating ceph-iscsi config... 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Traceback (most recent call last): 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: return _run_code(code, main_globals, None, 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: exec(code, run_globals) 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:15.609 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:15.610 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:15.610 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:15.610 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:15.610 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:16.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:16 smithi130 ceph-mon[26394]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T21:19:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:16 smithi130 ceph-mon[26394]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:19:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:19:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:19:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:19:16.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:19:16.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: return future.result(timeout) 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.__get_result() 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: raise self._exception 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:19:16.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:16.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:16.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:16.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:16.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:16.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:16.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:16 smithi130 ceph-mon[26394]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:19:16.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:16 smithi130 ceph-mon[26394]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:19:16.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:16 smithi130 ceph-mon[21867]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T21:19:16.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:16 smithi130 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:16.556 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:19:16.557 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:16.557 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:16.557 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:16.557 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:16.557 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:16.557 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:19:16.557 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:16.557 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:19:16.557 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:16.557 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:19:16.557 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:16.557 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:16.557 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:19:16.558 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: return future.result(timeout) 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.__get_result() 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: raise self._exception 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:16.559 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:16.561 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:16.561 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:16.561 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:16.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:16 smithi130 ceph-mon[21867]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:19:16.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:16 smithi130 ceph-mon[21867]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:19:16.588 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:19:16] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-04-22T21:19:16.589 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:16 smithi150 ceph-mon[22676]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T21:19:16.589 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:16 smithi150 ceph-mon[22676]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:16.589 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:19:16.589 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:16.589 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:16.589 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:16.589 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:16.590 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: return future.result(timeout) 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.__get_result() 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:19:16.591 INFO:journalctl@ceph.mon.b.smithi150.stdout: raise self._exception 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:16.592 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:19:16.593 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:16.593 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:19:16.593 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:16.593 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:16.593 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:16.593 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:16.593 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:16.593 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:16.593 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:16.593 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:16.593 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:16.593 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:16 smithi150 ceph-mon[22676]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:19:16.593 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:16 smithi150 ceph-mon[22676]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:19:18.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:18 smithi130 ceph-mon[26394]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:19:18.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:18 smithi130 ceph-mon[21867]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:19:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:18 smithi150 ceph-mon[22676]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:19:18.931 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 systemd[1]: Stopping Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:19:18.931 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 bash[45913]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T21:19:19.207 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:19:18.943Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T21:19:19.208 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:19:18.943Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T21:19:19.208 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:19:18.943Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T21:19:19.208 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:19:18.943Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T21:19:19.208 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:19:18.943Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T21:19:19.208 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:19:18.943Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T21:19:19.208 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:19:18.944Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T21:19:19.208 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:19:18.944Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T21:19:19.208 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:19:18.944Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T21:19:19.208 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:19:18.944Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T21:19:19.208 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:19:18.944Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T21:19:19.208 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[39560]: ts=2024-04-22T21:19:18.944Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T21:19:19.208 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:18 smithi150 podman[45926]: 2024-04-22 21:19:18.954123354 +0000 UTC m=+0.126010366 container died afc570412b78fd5f690739d7f29e7d0051980d22495614171d1e23fa2c47a0bc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:19.575 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:19 smithi150 podman[45926]: 2024-04-22 21:19:19.336039673 +0000 UTC m=+0.507926698 container cleanup afc570412b78fd5f690739d7f29e7d0051980d22495614171d1e23fa2c47a0bc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:19.575 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:19 smithi150 bash[45926]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T21:19:19.575 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:19 smithi150 bash[45960]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T21:19:19.843 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:19 smithi150 podman[45941]: 2024-04-22 21:19:19.587911794 +0000 UTC m=+0.634087109 container remove afc570412b78fd5f690739d7f29e7d0051980d22495614171d1e23fa2c47a0bc (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:19.844 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:19 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T21:19:19.844 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:19 smithi150 systemd[1]: Stopped Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:19:19.844 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:19 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.943s CPU time. 2024-04-22T21:19:19.844 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:19 smithi150 systemd[1]: Starting Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:19:19.844 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:19 smithi150 podman[46030]: 2024-04-22 21:19:19.720269811 +0000 UTC m=+0.019577006 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T21:19:19.844 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:19 smithi150 podman[46030]: 2024-04-22 21:19:19.856813016 +0000 UTC m=+0.156120203 container create 686c0cd096083f51c5256c6573e9769e45a875c3d6e013eff85973378a788e32 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:20.094 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 podman[46030]: 2024-04-22 21:19:20.075101652 +0000 UTC m=+0.374408848 container init 686c0cd096083f51c5256c6573e9769e45a875c3d6e013eff85973378a788e32 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:20.094 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 podman[46030]: 2024-04-22 21:19:20.077744797 +0000 UTC m=+0.377051996 container start 686c0cd096083f51c5256c6573e9769e45a875c3d6e013eff85973378a788e32 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:19:20.095 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.101Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T21:19:20.095 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.101Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T21:19:20.095 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.101Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T21:19:20.095 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.101Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi150 (none))" 2024-04-22T21:19:20.095 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.101Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T21:19:20.095 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.101Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T21:19:20.095 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.104Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T21:19:20.095 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.104Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T21:19:20.095 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.105Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T21:19:20.095 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.107Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T21:19:20.095 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.107Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.576µs 2024-04-22T21:19:20.095 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.107Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T21:19:20.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-mon[22676]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:19:20.355 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 bash[46030]: 686c0cd096083f51c5256c6573e9769e45a875c3d6e013eff85973378a788e32 2024-04-22T21:19:20.355 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 systemd[1]: Started Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:19:20.355 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.310Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-22T21:19:20.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:20 smithi130 ceph-mon[26394]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:19:20.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:20 smithi130 ceph-mon[21867]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:19:20.801 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:20 smithi150 systemd[1]: Stopping Ceph node-exporter.b for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:19:20.802 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.549Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-22T21:19:20.802 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.550Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-22T21:19:20.802 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.550Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=37.246µs wal_replay_duration=442.960889ms total_replay_duration=443.012852ms 2024-04-22T21:19:20.802 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.553Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T21:19:20.802 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.553Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T21:19:20.802 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.553Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T21:19:20.802 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.577Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.027946ms db_storage=889ns remote_storage=1.67µs web_handler=313ns query_engine=766ns scrape=554.887µs scrape_sd=115.698µs notify=10.996µs notify_sd=8.587µs rules=22.825397ms 2024-04-22T21:19:20.802 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:20.577Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T21:19:21.105 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:20 smithi150 bash[46154]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter.b" found: no such container 2024-04-22T21:19:21.105 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:20 smithi150 podman[46167]: 2024-04-22 21:19:20.939323668 +0000 UTC m=+0.116781472 container died aafb8379315730f2eed5b3877618269314d6c94695bd78fbde37c6a6ceeb9cb5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:19:21.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:21 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:21.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:21 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:21.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:21 smithi130 ceph-mon[26394]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-22T21:19:21.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:21 smithi130 ceph-mon[26394]: Reconfiguring daemon node-exporter.b on smithi150 2024-04-22T21:19:21.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:21 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:21.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:21 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:21.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:21 smithi130 ceph-mon[21867]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-22T21:19:21.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:21 smithi130 ceph-mon[21867]: Reconfiguring daemon node-exporter.b on smithi150 2024-04-22T21:19:21.584 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:21 smithi150 podman[46167]: 2024-04-22 21:19:21.345839384 +0000 UTC m=+0.523297176 container cleanup aafb8379315730f2eed5b3877618269314d6c94695bd78fbde37c6a6ceeb9cb5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:19:21.584 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:21 smithi150 bash[46167]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b 2024-04-22T21:19:21.584 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:21 smithi150 bash[46194]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter.b" found: no such container 2024-04-22T21:19:21.585 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:21 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:21.585 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:21 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:21.585 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:21 smithi150 ceph-mon[22676]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-22T21:19:21.585 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:21 smithi150 ceph-mon[22676]: Reconfiguring daemon node-exporter.b on smithi150 2024-04-22T21:19:21.843 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:21 smithi150 podman[46181]: 2024-04-22 21:19:21.596981179 +0000 UTC m=+0.657751551 container remove aafb8379315730f2eed5b3877618269314d6c94695bd78fbde37c6a6ceeb9cb5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:19:21.843 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:21 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-22T21:19:21.843 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:21 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-22T21:19:21.843 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:21 smithi150 systemd[1]: Stopped Ceph node-exporter.b for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:19:21.843 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:21 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 3.814s CPU time. 2024-04-22T21:19:21.844 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:21 smithi150 systemd[1]: Starting Ceph node-exporter.b for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:19:21.844 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:21 smithi150 podman[46262]: 2024-04-22 21:19:21.731513789 +0000 UTC m=+0.017432975 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-22T21:19:21.844 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:21 smithi150 podman[46262]: 2024-04-22 21:19:21.856305533 +0000 UTC m=+0.142224762 container create 2bcf914662579cf3ddad73910129323f0322ef7c2619466ee219b968f05c5931 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:19:22.105 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 podman[46262]: 2024-04-22 21:19:22.058580671 +0000 UTC m=+0.344499855 container init 2bcf914662579cf3ddad73910129323f0322ef7c2619466ee219b968f05c5931 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:19:22.105 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 podman[46262]: 2024-04-22 21:19:22.061141952 +0000 UTC m=+0.347061138 container start 2bcf914662579cf3ddad73910129323f0322ef7c2619466ee219b968f05c5931 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:19:22.105 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.064Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-22T21:19:22.105 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.064Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-22T21:19:22.105 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.064Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-22T21:19:22.106 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.064Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T21:19:22.106 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-22T21:19:22.106 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=arp 2024-04-22T21:19:22.106 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-22T21:19:22.106 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-22T21:19:22.106 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-22T21:19:22.106 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-22T21:19:22.106 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-22T21:19:22.106 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-22T21:19:22.106 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-22T21:19:22.106 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-22T21:19:22.107 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=edac 2024-04-22T21:19:22.107 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-22T21:19:22.107 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-22T21:19:22.107 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-22T21:19:22.107 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-22T21:19:22.107 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-22T21:19:22.107 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-22T21:19:22.108 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-22T21:19:22.108 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-22T21:19:22.108 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-22T21:19:22.108 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-22T21:19:22.108 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-22T21:19:22.108 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-22T21:19:22.108 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-22T21:19:22.108 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-22T21:19:22.108 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-22T21:19:22.108 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-22T21:19:22.108 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=os 2024-04-22T21:19:22.108 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-22T21:19:22.108 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=stat 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=time 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=uname 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-22T21:19:22.109 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-22T21:19:22.110 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[46276]: ts=2024-04-22T21:19:22.065Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-22T21:19:22.255 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:19:21 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[55940]: level=info ts=2024-04-22T21:19:21.883Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003437212s 2024-04-22T21:19:22.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:22 smithi130 ceph-mon[26394]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:19:22.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:22 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:22.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:22 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:22.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:22 smithi130 ceph-mon[21867]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:19:22.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:22 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:22.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:22 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:22.605 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 bash[46262]: 2bcf914662579cf3ddad73910129323f0322ef7c2619466ee219b968f05c5931 2024-04-22T21:19:22.605 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:19:22 smithi150 systemd[1]: Started Ceph node-exporter.b for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:19:22.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-mon[22676]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:19:22.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:22.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:22 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:23.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:23 smithi150 ceph-mon[22676]: Reconfiguring grafana.a (dependencies changed)... 2024-04-22T21:19:23.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:23 smithi150 ceph-mon[22676]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-22T21:19:23.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:23 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:23.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:23 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:23.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:23 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T21:19:23.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:23 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T21:19:23.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:23 smithi150 ceph-mon[22676]: Reconfiguring daemon grafana.a on smithi150 2024-04-22T21:19:23.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:23 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:19:23.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[26394]: Reconfiguring grafana.a (dependencies changed)... 2024-04-22T21:19:23.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[26394]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-22T21:19:23.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:23.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:23.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T21:19:23.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T21:19:23.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[26394]: Reconfiguring daemon grafana.a on smithi150 2024-04-22T21:19:23.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:19:23.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[21867]: Reconfiguring grafana.a (dependencies changed)... 2024-04-22T21:19:23.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[21867]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-22T21:19:23.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:23.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:23.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T21:19:23.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T21:19:23.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[21867]: Reconfiguring daemon grafana.a on smithi150 2024-04-22T21:19:23.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:23 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:19:24.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:24 smithi150 ceph-mon[22676]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:19:24.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:24 smithi130 ceph-mon[26394]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:19:24.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:24 smithi130 ceph-mon[21867]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:19:25.230 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:25 smithi150 systemd[1]: Stopping Ceph grafana.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:19:25.230 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:25 smithi150 bash[46449]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana.a" found: no such container 2024-04-22T21:19:25.522 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:25 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[38760]: t=2024-04-22T21:19:25+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-22T21:19:25.522 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:25 smithi150 podman[46462]: 2024-04-22 21:19:25.252995951 +0000 UTC m=+0.131880271 container died b3f3edd157587ed3f07491f9e834a0b3e563e659f50a0da8ffaad3391bcd1549 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, version=8.5, architecture=x86_64, release=236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-type=git, io.openshift.expose-services=) 2024-04-22T21:19:25.855 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:25 smithi150 podman[46462]: 2024-04-22 21:19:25.633846648 +0000 UTC m=+0.512730974 container cleanup b3f3edd157587ed3f07491f9e834a0b3e563e659f50a0da8ffaad3391bcd1549 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2, io.openshift.expose-services=, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, name=ubi8, architecture=x86_64, build-date=2022-03-28T10:36:18.413762, vendor=Red Hat, Inc., vcs-type=git, distribution-scope=public, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner ) 2024-04-22T21:19:25.855 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:25 smithi150 bash[46462]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a 2024-04-22T21:19:25.855 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:25 smithi150 bash[46488]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana.a" found: no such container 2024-04-22T21:19:26.223 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:25 smithi150 podman[46476]: 2024-04-22 21:19:25.968437941 +0000 UTC m=+0.711306745 container remove b3f3edd157587ed3f07491f9e834a0b3e563e659f50a0da8ffaad3391bcd1549 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, distribution-scope=public, vendor=Red Hat, Inc., io.buildah.version=1.24.2, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, name=ubi8, io.openshift.expose-services=, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, build-date=2022-03-28T10:36:18.413762, release=236.1648460182, vcs-type=git, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5) 2024-04-22T21:19:26.223 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:25 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-22T21:19:26.223 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:25 smithi150 systemd[1]: Stopped Ceph grafana.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:19:26.224 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:25 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@grafana.a.service: Consumed 3.317s CPU time. 2024-04-22T21:19:26.224 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:25 smithi150 systemd[1]: Starting Ceph grafana.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:19:26.224 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 podman[46557]: 2024-04-22 21:19:26.108541842 +0000 UTC m=+0.018181891 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-22T21:19:26.224 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 podman[46557]: 2024-04-22 21:19:26.236262383 +0000 UTC m=+0.145902433 container create e021384d6729b2adc301b98aa19981c5ae7b00e7cb411116ec5c20c28db490d5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vendor=Red Hat, Inc., io.buildah.version=1.24.2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, release=236.1648460182, maintainer=Paul Cuzner , description=Ceph Grafana Container, vcs-type=git, io.openshift.expose-services=, name=ubi8, build-date=2022-03-28T10:36:18.413762) 2024-04-22T21:19:26.498 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-mon[22676]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:19:26.498 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 podman[46557]: 2024-04-22 21:19:26.480758395 +0000 UTC m=+0.390398443 container init e021384d6729b2adc301b98aa19981c5ae7b00e7cb411116ec5c20c28db490d5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, release=236.1648460182, version=8.5, name=ubi8, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.tags=base rhel8, io.openshift.expose-services=, io.buildah.version=1.24.2, description=Ceph Grafana Container, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public) 2024-04-22T21:19:26.498 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 podman[46557]: 2024-04-22 21:19:26.483616602 +0000 UTC m=+0.393256659 container start e021384d6729b2adc301b98aa19981c5ae7b00e7cb411116ec5c20c28db490d5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, release=236.1648460182, version=8.5, name=ubi8, maintainer=Paul Cuzner , io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, build-date=2022-03-28T10:36:18.413762, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.expose-services=) 2024-04-22T21:19:26.588 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:26 smithi130 ceph-mon[21867]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:19:26.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:26 smithi130 ceph-mon[26394]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:19:26.768 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE Bus STOPPING 2024-04-22T21:19:26.768 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T21:19:26.768 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE Bus STOPPED 2024-04-22T21:19:26.769 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE Bus STARTING 2024-04-22T21:19:26.769 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE Serving on http://:::9283 2024-04-22T21:19:26.769 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE Bus STARTED 2024-04-22T21:19:26.769 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-04-22T21:19:26.769 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-04-22T21:19:26.769 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-04-22T21:19:26.769 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-04-22T21:19:26.770 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-22T21:19:26.770 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-22T21:19:26.770 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-22T21:19:26.770 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-22T21:19:26.770 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-04-22T21:19:26.770 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-04-22T21:19:26.770 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-04-22T21:19:26.770 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-04-22T21:19:26.770 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-04-22T21:19:26.770 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="App mode production" logger=settings 2024-04-22T21:19:26.770 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-04-22T21:19:26.771 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-22T21:19:26.771 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-04-22T21:19:26.771 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=684.056µs 2024-04-22T21:19:26.771 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-22T21:19:26.771 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-22T21:19:26.771 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-22T21:19:26.771 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 bash[46557]: e021384d6729b2adc301b98aa19981c5ae7b00e7cb411116ec5c20c28db490d5 2024-04-22T21:19:26.771 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 systemd[1]: Started Ceph grafana.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:19:26.771 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-22T21:19:26.771 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-22T21:19:26.771 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-22T21:19:26.771 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-04-22T21:19:26.771 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-22T21:19:26.772 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-04-22T21:19:26.772 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-22T21:19:26.772 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-22T21:19:26.772 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:19:26+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-22T21:19:27.105 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE Bus STOPPING 2024-04-22T21:19:27.105 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T21:19:27.105 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE Bus STOPPED 2024-04-22T21:19:27.105 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE Bus STARTING 2024-04-22T21:19:27.105 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE Serving on http://:::9283 2024-04-22T21:19:27.105 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE Bus STARTED 2024-04-22T21:19:27.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE Bus STOPPING 2024-04-22T21:19:27.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T21:19:27.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE Bus STOPPED 2024-04-22T21:19:27.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:26] ENGINE Bus STARTING 2024-04-22T21:19:27.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:27] ENGINE Serving on http://:::9283 2024-04-22T21:19:27.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:19:27] ENGINE Bus STARTED 2024-04-22T21:19:27.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:19:27.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:19:27.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T21:19:27.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T21:19:27.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:19:27.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:19:27.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2024-04-22T21:19:27.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2024-04-22T21:19:27.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:19:27.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:19:27.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150:3000"}]: dispatch 2024-04-22T21:19:27.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150:3000"}]: dispatch 2024-04-22T21:19:27.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2601733442' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:19:27.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1062111852' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2002813143"}]: dispatch 2024-04-22T21:19:27.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2002813143"}]: dispatch 2024-04-22T21:19:27.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:19:27.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:19:27.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T21:19:27.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T21:19:27.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:19:27.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:19:27.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2024-04-22T21:19:27.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2024-04-22T21:19:27.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:19:27.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:19:27.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150:3000"}]: dispatch 2024-04-22T21:19:27.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150:3000"}]: dispatch 2024-04-22T21:19:27.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2601733442' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:19:27.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1062111852' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2002813143"}]: dispatch 2024-04-22T21:19:27.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:27 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2002813143"}]: dispatch 2024-04-22T21:19:27.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150:3000"}]: dispatch 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150:3000"}]: dispatch 2024-04-22T21:19:27.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:27.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2601733442' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:19:27.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1062111852' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2002813143"}]: dispatch 2024-04-22T21:19:27.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:27 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2002813143"}]: dispatch 2024-04-22T21:19:29.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:28 smithi130 ceph-mon[26394]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:19:29.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:28 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2002813143"}]': finished 2024-04-22T21:19:29.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:28 smithi130 ceph-mon[26394]: osdmap e97: 8 total, 8 up, 8 in 2024-04-22T21:19:29.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:28 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1299231440' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/2780444297"}]: dispatch 2024-04-22T21:19:29.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:28 smithi130 ceph-mon[21867]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:19:29.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:28 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2002813143"}]': finished 2024-04-22T21:19:29.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:28 smithi130 ceph-mon[21867]: osdmap e97: 8 total, 8 up, 8 in 2024-04-22T21:19:29.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:28 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1299231440' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/2780444297"}]: dispatch 2024-04-22T21:19:29.068 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:28 smithi150 ceph-mon[22676]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:19:29.068 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:28 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2002813143"}]': finished 2024-04-22T21:19:29.068 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:28 smithi150 ceph-mon[22676]: osdmap e97: 8 total, 8 up, 8 in 2024-04-22T21:19:29.068 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:28 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1299231440' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/2780444297"}]: dispatch 2024-04-22T21:19:29.605 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:19:29] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.33.4" 2024-04-22T21:19:29.939 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:29 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1299231440' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/2780444297"}]': finished 2024-04-22T21:19:29.939 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:29 smithi150 ceph-mon[22676]: osdmap e98: 8 total, 8 up, 8 in 2024-04-22T21:19:29.939 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:29 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3309672584' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/840813444"}]: dispatch 2024-04-22T21:19:29.940 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:29 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/840813444"}]: dispatch 2024-04-22T21:19:29.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:29 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1299231440' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/2780444297"}]': finished 2024-04-22T21:19:29.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:29 smithi130 ceph-mon[26394]: osdmap e98: 8 total, 8 up, 8 in 2024-04-22T21:19:29.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:29 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3309672584' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/840813444"}]: dispatch 2024-04-22T21:19:29.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:29 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/840813444"}]: dispatch 2024-04-22T21:19:29.974 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:29 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1299231440' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/2780444297"}]': finished 2024-04-22T21:19:29.974 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:29 smithi130 ceph-mon[21867]: osdmap e98: 8 total, 8 up, 8 in 2024-04-22T21:19:29.974 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:29 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3309672584' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/840813444"}]: dispatch 2024-04-22T21:19:29.974 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:29 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/840813444"}]: dispatch 2024-04-22T21:19:31.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[21867]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:19:31.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/840813444"}]': finished 2024-04-22T21:19:31.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[21867]: osdmap e99: 8 total, 8 up, 8 in 2024-04-22T21:19:31.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2394467617' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1660076439"}]: dispatch 2024-04-22T21:19:31.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1660076439"}]: dispatch 2024-04-22T21:19:31.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:31.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:31.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[26394]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:19:31.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/840813444"}]': finished 2024-04-22T21:19:31.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[26394]: osdmap e99: 8 total, 8 up, 8 in 2024-04-22T21:19:31.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2394467617' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1660076439"}]: dispatch 2024-04-22T21:19:31.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1660076439"}]: dispatch 2024-04-22T21:19:31.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:31.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:30 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:30 smithi150 ceph-mon[22676]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:19:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:30 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/840813444"}]': finished 2024-04-22T21:19:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:30 smithi150 ceph-mon[22676]: osdmap e99: 8 total, 8 up, 8 in 2024-04-22T21:19:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:30 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2394467617' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1660076439"}]: dispatch 2024-04-22T21:19:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:30 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1660076439"}]: dispatch 2024-04-22T21:19:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:30 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:30 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:31.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1660076439"}]': finished 2024-04-22T21:19:31.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[26394]: osdmap e100: 8 total, 8 up, 8 in 2024-04-22T21:19:31.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1838471880' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3105135989"}]: dispatch 2024-04-22T21:19:31.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3105135989"}]: dispatch 2024-04-22T21:19:31.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:31.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:31.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:19:31.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:19:31.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:31.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:19:31.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:19:31.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:19:31.954 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1660076439"}]': finished 2024-04-22T21:19:31.954 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[21867]: osdmap e100: 8 total, 8 up, 8 in 2024-04-22T21:19:31.954 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1838471880' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3105135989"}]: dispatch 2024-04-22T21:19:31.954 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3105135989"}]: dispatch 2024-04-22T21:19:31.954 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:31.954 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:31.955 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:19:31.955 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:19:31.955 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:31.955 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:19:31.955 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:19:31.955 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:31 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:19:32.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:31 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1660076439"}]': finished 2024-04-22T21:19:32.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:31 smithi150 ceph-mon[22676]: osdmap e100: 8 total, 8 up, 8 in 2024-04-22T21:19:32.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:31 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1838471880' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3105135989"}]: dispatch 2024-04-22T21:19:32.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:31 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3105135989"}]: dispatch 2024-04-22T21:19:32.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:31 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:32.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:31 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:32.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:31 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:19:32.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:31 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:19:32.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:31 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:19:32.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:31 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:19:32.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:31 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:19:32.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:31 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:19:32.960 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:32 smithi130 ceph-mon[26394]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:19:32.960 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:32 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:19:32.960 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:32 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:19:32.960 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:32 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3105135989"}]': finished 2024-04-22T21:19:32.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:32 smithi130 ceph-mon[26394]: osdmap e101: 8 total, 8 up, 8 in 2024-04-22T21:19:32.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:32 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1786176289' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/2780444297"}]: dispatch 2024-04-22T21:19:32.961 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:32 smithi130 ceph-mon[21867]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:19:32.961 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:32 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:19:32.961 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:32 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:19:32.961 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:32 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3105135989"}]': finished 2024-04-22T21:19:32.961 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:32 smithi130 ceph-mon[21867]: osdmap e101: 8 total, 8 up, 8 in 2024-04-22T21:19:32.961 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:32 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1786176289' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/2780444297"}]: dispatch 2024-04-22T21:19:33.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:32 smithi150 ceph-mon[22676]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:19:33.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:32 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:19:33.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:32 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:19:33.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:32 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3105135989"}]': finished 2024-04-22T21:19:33.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:32 smithi150 ceph-mon[22676]: osdmap e101: 8 total, 8 up, 8 in 2024-04-22T21:19:33.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:32 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1786176289' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/2780444297"}]: dispatch 2024-04-22T21:19:33.855 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi150\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi150\", job=\"node\", machine=\"x86_64\", nodename=\"smithi150\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi150\", job=\"node\", machine=\"x86_64\", nodename=\"smithi150\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:19:33.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:33 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1786176289' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/2780444297"}]': finished 2024-04-22T21:19:33.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:33 smithi150 ceph-mon[22676]: osdmap e102: 8 total, 8 up, 8 in 2024-04-22T21:19:33.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:33 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4103438303' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/995961703"}]: dispatch 2024-04-22T21:19:33.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:33 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/995961703"}]: dispatch 2024-04-22T21:19:33.982 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:33 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1786176289' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/2780444297"}]': finished 2024-04-22T21:19:33.982 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:33 smithi130 ceph-mon[21867]: osdmap e102: 8 total, 8 up, 8 in 2024-04-22T21:19:33.982 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:33 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4103438303' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/995961703"}]: dispatch 2024-04-22T21:19:33.982 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:33 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/995961703"}]: dispatch 2024-04-22T21:19:33.982 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:33 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1786176289' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/2780444297"}]': finished 2024-04-22T21:19:33.982 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:33 smithi130 ceph-mon[26394]: osdmap e102: 8 total, 8 up, 8 in 2024-04-22T21:19:33.982 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:33 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4103438303' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/995961703"}]: dispatch 2024-04-22T21:19:33.983 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:33 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/995961703"}]: dispatch 2024-04-22T21:19:35.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:34 smithi130 ceph-mon[21867]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:19:35.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:34 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/995961703"}]': finished 2024-04-22T21:19:35.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:34 smithi130 ceph-mon[21867]: osdmap e103: 8 total, 8 up, 8 in 2024-04-22T21:19:35.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:34 smithi130 ceph-mon[26394]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:19:35.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:34 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/995961703"}]': finished 2024-04-22T21:19:35.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:34 smithi130 ceph-mon[26394]: osdmap e103: 8 total, 8 up, 8 in 2024-04-22T21:19:35.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-mon[22676]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:19:35.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/995961703"}]': finished 2024-04-22T21:19:35.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-mon[22676]: osdmap e103: 8 total, 8 up, 8 in 2024-04-22T21:19:35.105 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:19:34.669+0000 7f40299f7700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:35.105 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Creating ceph-iscsi config... 2024-04-22T21:19:35.105 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:35.105 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:35.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:35.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:35.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:35.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Traceback (most recent call last): 2024-04-22T21:19:35.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:35.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: return _run_code(code, main_globals, None, 2024-04-22T21:19:35.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:35.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: exec(code, run_globals) 2024-04-22T21:19:35.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:35.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:35.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:35.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:35.106 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:35.107 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:35.107 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:35.107 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:35.107 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:35.107 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Traceback (most recent call last): 2024-04-22T21:19:35.107 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:19:35.107 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:19:35.108 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:19:35.108 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:19:35.108 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:19:35.108 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:19:35.108 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:19:35.108 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: return future.result(timeout) 2024-04-22T21:19:35.108 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:19:35.108 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: return self.__get_result() 2024-04-22T21:19:35.108 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:19:35.108 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: raise self._exception 2024-04-22T21:19:35.108 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:19:35.108 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: use_current_daemon_image=reconfig, 2024-04-22T21:19:35.108 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Creating ceph-iscsi config... 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: Traceback (most recent call last): 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: return _run_code(code, main_globals, None, 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: exec(code, run_globals) 2024-04-22T21:19:35.109 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:35.110 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:35.110 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:35.110 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:35.110 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:35.110 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:35.110 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:35.110 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:35.110 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:36.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:35 smithi130 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:36.051 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:19:36.051 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:36.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:36.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:36.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:36.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:36.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:19:36.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:36.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:19:36.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:36.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:19:36.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:36.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:36.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:36.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:36.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:36.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:36.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:36.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:36.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:36.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:19:36.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:19:36.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:19:36.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:19:36.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:19:36.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:19:36.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: return future.result(timeout) 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.__get_result() 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: raise self._exception 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:36.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:36.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:36.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:36.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:36.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:19:36.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:36.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:19:36.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:36.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:19:36.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:36.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:36.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:36.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:36.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:36.056 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:36.056 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:36.056 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:36.056 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:36.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:35 smithi130 ceph-mon[26394]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:36.056 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:19:36.056 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:36.056 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:36.056 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:36.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:36.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:36.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:19:36.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:36.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:19:36.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:36.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:19:36.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:36.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:36.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:19:36.058 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:19:36.059 INFO:journalctl@ceph.mon.c.smithi130.stdout: return future.result(timeout) 2024-04-22T21:19:36.059 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:19:36.059 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.__get_result() 2024-04-22T21:19:36.059 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:19:36.059 INFO:journalctl@ceph.mon.c.smithi130.stdout: raise self._exception 2024-04-22T21:19:36.059 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:19:36.059 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:19:36.059 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:19:36.060 INFO:journalctl@ceph.mon.c.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:19:36.060 INFO:journalctl@ceph.mon.c.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:36.060 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:19:36.061 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:36.061 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:36.061 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:36.061 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:36.061 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:36.061 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:19:36.061 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:36.061 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:19:36.061 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:36.061 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:19:36.061 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:36.061 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:36.062 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:36.062 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:36.062 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:36.062 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:36.062 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:36.062 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:36.062 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:36.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:35 smithi150 ceph-mon[22676]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:36.105 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:19:36.105 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:36.105 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:36.105 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:36.105 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:36.105 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:36.105 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:19:36.105 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:36.105 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:19:36.105 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:19:36.106 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: return future.result(timeout) 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.__get_result() 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: raise self._exception 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:19:36.107 INFO:journalctl@ceph.mon.b.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:19:36.108 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:19:36.109 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:19:36.109 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:19:36.109 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:19:36.109 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:19:36.109 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:19:36.109 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:19:37.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:36 smithi130 ceph-mon[26394]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T21:19:37.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:36 smithi130 ceph-mon[21867]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T21:19:37.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:36 smithi150 ceph-mon[22676]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T21:19:39.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:38 smithi130 ceph-mon[26394]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:19:39.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:38 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:19:39.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:38 smithi130 ceph-mon[21867]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:19:39.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:38 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:19:39.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:38 smithi150 ceph-mon[22676]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:19:39.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:38 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:19:39.105 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:19:39.855 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:19:39] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-04-22T21:19:40.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:40 smithi150 ceph-mon[22676]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 681 B/s rd, 0 op/s 2024-04-22T21:19:40.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:40 smithi130 ceph-mon[26394]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 681 B/s rd, 0 op/s 2024-04-22T21:19:40.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:40 smithi130 ceph-mon[21867]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 681 B/s rd, 0 op/s 2024-04-22T21:19:42.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:42 smithi130 ceph-mon[26394]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-04-22T21:19:42.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:42 smithi130 ceph-mon[21867]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-04-22T21:19:42.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:42 smithi150 ceph-mon[22676]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-04-22T21:19:43.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:43.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:19:44.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:44 smithi130 ceph-mon[21867]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:19:44.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:44 smithi130 ceph-mon[26394]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:19:44.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:44 smithi150 ceph-mon[22676]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:19:46.301 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:46 smithi130 ceph-mon[26394]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 445 B/s rd, 0 op/s 2024-04-22T21:19:46.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:46 smithi130 ceph-mon[21867]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 445 B/s rd, 0 op/s 2024-04-22T21:19:46.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:46 smithi150 ceph-mon[22676]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 445 B/s rd, 0 op/s 2024-04-22T21:19:47.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:47 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4126560903' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:19:47.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:47 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4126560903' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:19:47.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:47 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4126560903' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:19:48.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:48 smithi130 ceph-mon[21867]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:19:48.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:48 smithi130 ceph-mon[26394]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:19:48.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:48 smithi150 ceph-mon[22676]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:19:49.105 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:48 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:48.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:19:49.854 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:49 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:19:49] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-04-22T21:19:50.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:50 smithi150 ceph-mon[22676]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:19:50.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:50 smithi130 ceph-mon[26394]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:19:50.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:50 smithi130 ceph-mon[21867]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:19:52.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:52 smithi130 ceph-mon[21867]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:19:52.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:52 smithi130 ceph-mon[26394]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:19:52.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:52 smithi150 ceph-mon[22676]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:19:53.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:53 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:53.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:19:54.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:54 smithi130 ceph-mon[21867]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:19:54.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:19:54.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:54 smithi130 ceph-mon[26394]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:19:54.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:19:54.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:54 smithi150 ceph-mon[22676]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:19:54.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:19:56.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:56 smithi130 ceph-mon[21867]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:19:56.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:56 smithi130 ceph-mon[26394]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:19:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:56 smithi150 ceph-mon[22676]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:19:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:57 smithi130 ceph-mon[21867]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:19:57.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:57 smithi130 ceph-mon[26394]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:19:57.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:57 smithi150 ceph-mon[22676]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:19:58.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:19:58 smithi130 ceph-mon[21867]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:19:58.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:19:58 smithi130 ceph-mon[26394]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:19:58.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:19:58 smithi150 ceph-mon[22676]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:19:59.105 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:19:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:19:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:19:59.854 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:19:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:19:59] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-04-22T21:20:00.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:00 smithi130 ceph-mon[26394]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:00.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:00 smithi130 ceph-mon[26394]: overall HEALTH_OK 2024-04-22T21:20:00.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:00 smithi130 ceph-mon[21867]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:00.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:00 smithi130 ceph-mon[21867]: overall HEALTH_OK 2024-04-22T21:20:00.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:00 smithi150 ceph-mon[22676]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:00.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:00 smithi150 ceph-mon[22676]: overall HEALTH_OK 2024-04-22T21:20:02.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:02 smithi130 ceph-mon[26394]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:02.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:02 smithi130 ceph-mon[21867]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:02.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:02 smithi150 ceph-mon[22676]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:03.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:20:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:20:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:20:04.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:04 smithi130 ceph-mon[26394]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:04.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:04 smithi130 ceph-mon[21867]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:04.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:04 smithi150 ceph-mon[22676]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:06.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:06 smithi130 ceph-mon[21867]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:06.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:06 smithi130 ceph-mon[26394]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:06.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:06 smithi150 ceph-mon[22676]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:07.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:07 smithi130 ceph-mon[26394]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:07.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:07 smithi130 ceph-mon[21867]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:07.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:07 smithi150 ceph-mon[22676]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:08.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:08 smithi130 ceph-mon[26394]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:08.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:08 smithi130 ceph-mon[21867]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:08 smithi150 ceph-mon[22676]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:09.104 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:20:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:20:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:20:09.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:09 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:20:09.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:09 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:20:09.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:09 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:20:09.605 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:20:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:20:09] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-04-22T21:20:10.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:10 smithi150 ceph-mon[22676]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:10.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:10 smithi130 ceph-mon[21867]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:10.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:10 smithi130 ceph-mon[26394]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:12.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:12 smithi150 ceph-mon[22676]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:12.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:12 smithi130 ceph-mon[26394]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:12.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:12 smithi130 ceph-mon[21867]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:13.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:20:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:20:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:20:14.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:14 smithi150 ceph-mon[22676]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:14.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:14 smithi130 ceph-mon[21867]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:14.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:14 smithi130 ceph-mon[26394]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:16.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:16 smithi130 ceph-mon[21867]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:16.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:16 smithi130 ceph-mon[26394]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:16.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:16 smithi150 ceph-mon[22676]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:17.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:17 smithi150 ceph-mon[22676]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:17.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:17 smithi130 ceph-mon[21867]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:17.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:17 smithi130 ceph-mon[26394]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:18 smithi150 ceph-mon[22676]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:18.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:18 smithi130 ceph-mon[21867]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:18.702 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:18 smithi130 ceph-mon[26394]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:19.104 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:20:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:20:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:20:19.854 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:20:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:20:19] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-04-22T21:20:20.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:20 smithi130 ceph-mon[21867]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:20.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:20 smithi130 ceph-mon[26394]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:20.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:20 smithi150 ceph-mon[22676]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:22.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:22 smithi130 ceph-mon[21867]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:22.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:22 smithi130 ceph-mon[26394]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:22.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:22 smithi150 ceph-mon[22676]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:23.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:20:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:20:23.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:20:24.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:24 smithi130 ceph-mon[21867]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:24.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:24 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:20:24.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:24 smithi130 ceph-mon[26394]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:24.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:24 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:20:24.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:24 smithi150 ceph-mon[22676]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:24.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:24 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:20:26.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:26 smithi130 ceph-mon[21867]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:26.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:26 smithi130 ceph-mon[26394]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:26.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:26 smithi150 ceph-mon[22676]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:27.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:27 smithi130 ceph-mon[21867]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:27.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:27 smithi130 ceph-mon[26394]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:27.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:27 smithi150 ceph-mon[22676]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:28.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:28 smithi130 ceph-mon[26394]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:28.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:28 smithi130 ceph-mon[21867]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:28.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:28 smithi150 ceph-mon[22676]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:29.104 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:20:28 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:20:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:20:29.854 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:20:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:20:29] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-04-22T21:20:30.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:30 smithi130 ceph-mon[21867]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:30.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:30 smithi130 ceph-mon[26394]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:30.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:30 smithi150 ceph-mon[22676]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:32.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:32 smithi130 ceph-mon[21867]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:32.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:32 smithi130 ceph-mon[26394]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:32.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:32 smithi150 ceph-mon[22676]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:33.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:20:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:20:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:20:34.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:34 smithi130 ceph-mon[21867]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:34.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:34 smithi130 ceph-mon[26394]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:34.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:34 smithi150 ceph-mon[22676]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:36.461 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:36 smithi130 ceph-mon[26394]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:36.462 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:36 smithi130 ceph-mon[21867]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:36.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:36 smithi150 ceph-mon[22676]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:37.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:37 smithi130 ceph-mon[26394]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:37.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:37 smithi130 ceph-mon[21867]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:37.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:37 smithi150 ceph-mon[22676]: from='client.15168 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:38.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:38 smithi150 ceph-mon[22676]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:38.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:38 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:38.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:38 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:38.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:38 smithi130 ceph-mon[26394]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:38.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:38 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:38.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:38 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:38.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:38 smithi130 ceph-mon[21867]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:38.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:38 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:38.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:38 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:38.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:20:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:20:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:20:39.604 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:20:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:20:39] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-04-22T21:20:39.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:39 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:20:39.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:39 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:39.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:39 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:39.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:39 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:20:39.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:39 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:39.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:39 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:39.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:39 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:20:39.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:39 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:39.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:39 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:40.441 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[21867]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:40.441 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:20:40.441 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:20:40.441 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:40.441 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:20:40.442 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:20:40.442 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:20:40.442 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:20:40.442 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:20:40.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[26394]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:40.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:20:40.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:20:40.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:40.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:20:40.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:20:40.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:20:40.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:20:40.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:40 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:20:40.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:40 smithi150 ceph-mon[22676]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:40.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:40 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:20:40.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:40 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:20:40.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:40 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:40.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:40 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:20:40.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:40 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:20:40.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:40 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:20:40.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:40 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:20:40.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:40 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:20:42.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:42 smithi130 ceph-mon[21867]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:42.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:42 smithi130 ceph-mon[26394]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:42.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:42 smithi150 ceph-mon[22676]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:43.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:20:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:20:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:20:44.465 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:44 smithi130 ceph-mon[21867]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:44.465 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:44 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:44.465 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:44 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:44.465 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:44 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:20:44.465 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:44 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:20:44.465 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:44 smithi130 ceph-mon[26394]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:44.465 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:44 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:44.466 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:44 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:44.466 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:44 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:20:44.466 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:44 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:20:44.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:44 smithi150 ceph-mon[22676]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:44.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:44 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:44.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:44 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:44.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:44 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:20:44.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:44 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:20:45.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:45 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:20:45.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:45 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:20:45.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:45 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/74773765' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:20:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:45 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:20:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:45 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:20:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:45 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/74773765' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:20:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:45 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:20:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:45 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:20:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:45 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/74773765' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:20:46.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:46 smithi150 ceph-mon[22676]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:46.660 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:46 smithi130 ceph-mon[21867]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:46.661 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:46 smithi130 ceph-mon[26394]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:48.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:20:48 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:20:48.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:20:48.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:48 smithi150 ceph-mon[22676]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:20:48.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:48 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:48.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:48 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:48.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:48 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:48.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:48 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:48.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:48 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:20:48.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:48 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:20:48.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:48 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:49.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[21867]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:20:49.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:49.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:49.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:49.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:49.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:20:49.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:20:49.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:49.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[26394]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:20:49.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:49.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:49.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:49.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:49.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:20:49.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:20:49.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:48 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:20:49.854 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:20:49 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:20:49] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-04-22T21:20:50.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:50 smithi130 ceph-mon[21867]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:50.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:50 smithi130 ceph-mon[26394]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:50 smithi150 ceph-mon[22676]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:52.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:52 smithi130 ceph-mon[26394]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:20:52.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:52 smithi130 ceph-mon[21867]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:20:52.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:52 smithi150 ceph-mon[22676]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:20:53.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:20:53 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:20:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:20:54.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:54 smithi130 ceph-mon[26394]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:20:54.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:20:54.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:54 smithi130 ceph-mon[21867]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:20:54.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:20:54.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:54 smithi150 ceph-mon[22676]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:20:54.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:20:55.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:55 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:55.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:55 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:55.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:55 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:20:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:56 smithi150 ceph-mon[22676]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:56.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:56 smithi130 ceph-mon[26394]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:56.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:56 smithi130 ceph-mon[21867]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:20:58.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:20:58 smithi150 ceph-mon[22676]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:58.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:20:58 smithi130 ceph-mon[21867]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:58.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:20:58 smithi130 ceph-mon[26394]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:20:58.905 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:20:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:20:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:20:59.854 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:20:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:20:59] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-04-22T21:21:00.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:00 smithi130 ceph-mon[26394]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:00.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:00 smithi130 ceph-mon[21867]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:00.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:00 smithi150 ceph-mon[22676]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:02.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:02 smithi130 ceph-mon[26394]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:02.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:02 smithi130 ceph-mon[21867]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:02.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:02 smithi150 ceph-mon[22676]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:03.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:21:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:21:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:21:04.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:04 smithi130 ceph-mon[26394]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:04.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:04 smithi130 ceph-mon[21867]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:04.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:04 smithi150 ceph-mon[22676]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:05.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:05 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:05.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:05 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:05.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:05 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:06.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:06 smithi150 ceph-mon[22676]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:06.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:06 smithi130 ceph-mon[21867]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:06.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:06 smithi130 ceph-mon[26394]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:08 smithi150 ceph-mon[22676]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:08.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:08 smithi130 ceph-mon[21867]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:08 smithi130 ceph-mon[26394]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:09.104 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:21:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:21:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:21:09.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:09 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:21:09.604 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:21:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:21:09] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-22T21:21:09.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:09 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:21:09.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:09 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:21:10.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:10 smithi150 ceph-mon[22676]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:10.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:10 smithi130 ceph-mon[26394]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:10.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:10 smithi130 ceph-mon[21867]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:12.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:12 smithi150 ceph-mon[22676]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:12.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:12 smithi130 ceph-mon[21867]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:12.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:12 smithi130 ceph-mon[26394]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:13.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:21:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:21:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:21:14.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:14 smithi130 ceph-mon[21867]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:14.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:14 smithi130 ceph-mon[26394]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:14.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:14 smithi150 ceph-mon[22676]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:15.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:15 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:15.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:15 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:15.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:15 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:16.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:16 smithi150 ceph-mon[22676]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:16.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:16 smithi130 ceph-mon[21867]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:16.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:16 smithi130 ceph-mon[26394]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:18 smithi150 ceph-mon[22676]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:18 smithi130 ceph-mon[21867]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:18.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:18 smithi130 ceph-mon[26394]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:19.104 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:21:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:21:18.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:21:19.854 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:21:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:21:19] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-22T21:21:20.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:20 smithi130 ceph-mon[21867]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:20.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:20 smithi130 ceph-mon[26394]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:20.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:20 smithi150 ceph-mon[22676]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:22.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:22 smithi130 ceph-mon[26394]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:22.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:22 smithi130 ceph-mon[21867]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:22.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:22 smithi150 ceph-mon[22676]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:23.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:21:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:21:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:21:24.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:24 smithi130 ceph-mon[26394]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:24.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:24 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:21:24.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:24 smithi130 ceph-mon[21867]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:24.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:24 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:21:24.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:24 smithi150 ceph-mon[22676]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:24.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:24 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:21:25.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:25 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:25.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:25 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:25.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:25 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:26.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:26 smithi150 ceph-mon[22676]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:26.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:26 smithi130 ceph-mon[26394]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:26.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:26 smithi130 ceph-mon[21867]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:28.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:28 smithi150 ceph-mon[22676]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:28.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:28 smithi130 ceph-mon[21867]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:28.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:28 smithi130 ceph-mon[26394]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:29.052 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:21:28 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:21:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:21:29.854 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:21:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:21:29] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-22T21:21:30.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:30 smithi130 ceph-mon[26394]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:30.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:30 smithi130 ceph-mon[21867]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:30.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:30 smithi150 ceph-mon[22676]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:32.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:32 smithi150 ceph-mon[22676]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:32.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:32 smithi130 ceph-mon[26394]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:32.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:32 smithi130 ceph-mon[21867]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:33.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:21:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:21:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:21:34.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:34 smithi130 ceph-mon[26394]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:34.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:34 smithi130 ceph-mon[21867]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:34.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:34 smithi150 ceph-mon[22676]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:35.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:35 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:35.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:35 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:35.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:35 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:36.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:36 smithi150 ceph-mon[22676]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:36.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:36 smithi130 ceph-mon[21867]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:36.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:36 smithi130 ceph-mon[26394]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:38.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:38 smithi150 ceph-mon[22676]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:38.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:38 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:21:38.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:38 smithi130 ceph-mon[26394]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:38.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:38 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:21:38.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:38 smithi130 ceph-mon[21867]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:38.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:38 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:21:39.104 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:21:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:21:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:21:39.854 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:21:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:21:39] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-04-22T21:21:40.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:40 smithi130 ceph-mon[26394]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:40.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:40 smithi130 ceph-mon[21867]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:40.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:40 smithi150 ceph-mon[22676]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:42.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:42 smithi150 ceph-mon[22676]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:42.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:42 smithi130 ceph-mon[26394]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:42.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:42 smithi130 ceph-mon[21867]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:43.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:21:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:21:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:21:44.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:44 smithi130 ceph-mon[26394]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:44.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:44 smithi130 ceph-mon[21867]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:44.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:44 smithi150 ceph-mon[22676]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:45 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:45.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:45 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:45.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:45 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:46.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:46 smithi150 ceph-mon[22676]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:46.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:46 smithi130 ceph-mon[26394]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:46.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:46 smithi130 ceph-mon[21867]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:48.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:48 smithi150 ceph-mon[22676]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:48.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:48 smithi130 ceph-mon[21867]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:48.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:48 smithi130 ceph-mon[26394]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:49.104 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:21:48 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:21:48.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:21:49.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:21:49 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:21:49] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-04-22T21:21:50.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:50 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:21:50.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:50 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:21:50.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:50 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:50.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:50 smithi130 ceph-mon[26394]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:50.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:50 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:21:50.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:50 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:21:50.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:50 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:50.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:50 smithi130 ceph-mon[21867]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:50 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:21:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:50 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:21:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:50 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:50 smithi150 ceph-mon[22676]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:52.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:52 smithi150 ceph-mon[22676]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:52.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:52 smithi130 ceph-mon[21867]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:52.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:52 smithi130 ceph-mon[26394]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:53.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:21:53 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:21:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:21:54.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:54 smithi130 ceph-mon[26394]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:54.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:54 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:21:54.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:54 smithi130 ceph-mon[21867]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:54.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:54 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:21:54.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:54 smithi150 ceph-mon[22676]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:54.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:54 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:21:55.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:55 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:55.740 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:55 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:55.740 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:55 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:21:56.492 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:56 smithi130 ceph-mon[21867]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:56.492 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:56 smithi130 ceph-mon[26394]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:56.493 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-22T21:21:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:56 smithi150 ceph-mon[22676]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:21:58.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:58 smithi150 ceph-mon[22676]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:58.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:58 smithi130 ceph-mon[21867]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:58.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:58 smithi130 ceph-mon[26394]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:21:58.883 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled to redeploy mgr.y on host 'smithi130' 2024-04-22T21:21:59.103 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:21:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:21:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:21:59.523 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3337586956' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T21:21:59.524 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[21867]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:21:59.524 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:59.524 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[21867]: Schedule redeploy daemon mgr.y 2024-04-22T21:21:59.524 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:59.524 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:59.524 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:21:59.524 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:21:59.524 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:59.524 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:21:59.524 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:21:59.524 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:21:59.524 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:21:59.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3337586956' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T21:21:59.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[26394]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:21:59.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:59.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[26394]: Schedule redeploy daemon mgr.y 2024-04-22T21:21:59.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:59.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:59.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:21:59.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:21:59.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:59.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:21:59.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:21:59.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:21:59.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:21:59 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:21:59.548 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps --refresh' 2024-04-22T21:21:59.604 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:21:59] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-22T21:21:59.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3337586956' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T21:21:59.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-mon[22676]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:21:59.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:59.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-mon[22676]: Schedule redeploy daemon mgr.y 2024-04-22T21:21:59.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:59.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:59.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:21:59.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:21:59.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:21:59.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:21:59.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:21:59.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:21:59.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:21:59 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:22:00.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:00 smithi130 ceph-mon[21867]: Deploying daemon mgr.y on smithi130 2024-04-22T21:22:00.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:00 smithi130 ceph-mon[21867]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:00.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:00 smithi130 ceph-mon[26394]: Deploying daemon mgr.y on smithi130 2024-04-22T21:22:00.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:00 smithi130 ceph-mon[26394]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:00.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:00 smithi150 ceph-mon[22676]: Deploying daemon mgr.y on smithi130 2024-04-22T21:22:00.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:00 smithi150 ceph-mon[22676]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:01.680 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:22:01.680 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (2m) 73s ago 10m 15.5M - 0.23.0 ba2b418f427c 23fd04f877ce 2024-04-22T21:22:01.680 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (2m) 73s ago 10m 40.4M - 8.3.5 dad864ee21e9 e021384d6729 2024-04-22T21:22:01.681 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (78s) 73s ago 9m 51.9M - 3.5 e1d6a67b021e 0a30d8841db2 2024-04-22T21:22:01.681 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283 running (5m) 73s ago 15m 508M - 19.0.0-3198-g56e81df3 82086b19bb07 327605fe868f 2024-04-22T21:22:01.681 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:9283 running (17m) 73s ago 17m 412M - 17.2.0 e1d6a67b021e 878d76986ae8 2024-04-22T21:22:01.681 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (17m) 73s ago 17m 58.8M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:22:01.681 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (15m) 73s ago 15m 45.1M 2048M 17.2.0 e1d6a67b021e 861909a68cf9 2024-04-22T21:22:01.681 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (15m) 73s ago 15m 44.7M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:22:01.681 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (2m) 73s ago 11m 15.0M - 1.3.1 1dbe0e931976 4057e105dc28 2024-04-22T21:22:01.681 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (2m) 73s ago 11m 9.83M - 1.3.1 1dbe0e931976 2bcf91466257 2024-04-22T21:22:01.681 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (15m) 73s ago 15m 52.5M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:22:01.681 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (14m) 73s ago 14m 54.5M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:22:01.681 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (14m) 73s ago 14m 47.9M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:22:01.681 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (13m) 73s ago 13m 50.7M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:22:01.682 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (13m) 73s ago 13m 51.2M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:22:01.682 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (13m) 73s ago 13m 50.2M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:22:01.682 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (12m) 73s ago 12m 51.5M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:22:01.682 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (12m) 73s ago 12m 50.9M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:22:01.682 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (2m) 73s ago 11m 60.5M - 2.33.4 514e6a882f6e 686c0cd09608 2024-04-22T21:22:01.682 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (9m) 73s ago 9m 73.8M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:22:01.682 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (9m) 73s ago 9m 71.2M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:22:02.482 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:02 smithi130 ceph-mon[21867]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:02.483 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:02 smithi130 ceph-mon[21867]: from='client.15219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:22:02.483 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:02 smithi130 ceph-mon[26394]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:02.483 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:02 smithi130 ceph-mon[26394]: from='client.15219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:22:02.484 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'sleep 180' 2024-04-22T21:22:02.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:02 smithi150 ceph-mon[22676]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:02.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:02 smithi150 ceph-mon[22676]: from='client.15219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:22:03.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:22:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:22:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:22:04.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:04 smithi130 ceph-mon[21867]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:04.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:04 smithi130 ceph-mon[26394]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:04.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:04 smithi150 ceph-mon[22676]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:05.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:05 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:05.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:05 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:05.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:05 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:06.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:06 smithi150 ceph-mon[22676]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T21:22:06.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:06 smithi130 ceph-mon[21867]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T21:22:06.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:06 smithi130 ceph-mon[26394]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T21:22:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:08 smithi150 ceph-mon[22676]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T21:22:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:08 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:22:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:08 smithi130 ceph-mon[26394]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T21:22:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:08 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:22:08.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:08 smithi130 ceph-mon[21867]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T21:22:08.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:08 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:22:09.103 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:22:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:22:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:22:09.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:22:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:22:09] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-04-22T21:22:10.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:10 smithi130 ceph-mon[21867]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-22T21:22:10.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:10 smithi130 ceph-mon[26394]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-22T21:22:10.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:10 smithi150 ceph-mon[22676]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-22T21:22:12.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:12 smithi150 ceph-mon[22676]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T21:22:12.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:12 smithi130 ceph-mon[21867]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T21:22:12.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:12 smithi130 ceph-mon[26394]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T21:22:13.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:22:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:22:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:22:14.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:14 smithi130 ceph-mon[21867]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T21:22:14.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:14 smithi130 ceph-mon[26394]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T21:22:14.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:14 smithi150 ceph-mon[22676]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T21:22:15.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:15 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:15.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:15 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:15.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:15 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:16.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:16 smithi150 ceph-mon[22676]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-22T21:22:16.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:16 smithi130 ceph-mon[21867]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-22T21:22:16.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:16 smithi130 ceph-mon[26394]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-22T21:22:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:18 smithi150 ceph-mon[22676]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T21:22:18.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:18 smithi130 ceph-mon[21867]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T21:22:18.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:18 smithi130 ceph-mon[26394]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T21:22:19.103 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:22:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:22:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:22:19.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:22:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:22:19] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-04-22T21:22:20.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:20 smithi130 ceph-mon[21867]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T21:22:20.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:20 smithi130 ceph-mon[26394]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T21:22:20.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:20 smithi150 ceph-mon[22676]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T21:22:22.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:22 smithi150 ceph-mon[22676]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:22.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:22 smithi130 ceph-mon[26394]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:22.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:22 smithi130 ceph-mon[21867]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:23.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:23 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:22:23.604 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:22:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:22:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:22:23.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:23 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:22:23.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:23 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:22:24.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:24 smithi130 ceph-mon[21867]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:24.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:24 smithi130 ceph-mon[26394]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:24.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:24 smithi150 ceph-mon[22676]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:25.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:25 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:25.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:25 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:25.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:25 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:26.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:26 smithi150 ceph-mon[22676]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:26.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:26 smithi130 ceph-mon[21867]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:26.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:26 smithi130 ceph-mon[26394]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:28.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:28 smithi150 ceph-mon[22676]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:28.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:28 smithi130 ceph-mon[21867]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:28.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:28 smithi130 ceph-mon[26394]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:29.103 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:22:28 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:22:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:22:29.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:22:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:22:29] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-04-22T21:22:30.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:30 smithi130 ceph-mon[21867]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:30.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:30 smithi130 ceph-mon[26394]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:30.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:30 smithi150 ceph-mon[22676]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:32.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:32 smithi150 ceph-mon[22676]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:32.790 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:32 smithi130 ceph-mon[21867]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:32.791 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:32 smithi130 ceph-mon[26394]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:33.370 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:33 smithi130 systemd[1]: Stopping Ceph mgr.y for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:22:33.632 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:33 smithi130 podman[59936]: 2024-04-22 21:22:33.384641436 +0000 UTC m=+0.152325182 container died 878d76986ae8f5d27124c0276ca0fa6636a69e8699f3a5fa9655ca8d8cfda59f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.buildah.version=1.19.8, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, architecture=x86_64, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_CLEAN=True) 2024-04-22T21:22:33.853 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:22:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:22:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:22:34.007 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:33 smithi130 podman[59936]: 2024-04-22 21:22:33.753852108 +0000 UTC m=+0.521535851 container cleanup 878d76986ae8f5d27124c0276ca0fa6636a69e8699f3a5fa9655ca8d8cfda59f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.component=centos-stream-container, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, GIT_CLEAN=True, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_BRANCH=HEAD, name=centos-stream, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, version=8) 2024-04-22T21:22:34.007 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:33 smithi130 bash[59936]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y 2024-04-22T21:22:34.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 podman[59950]: 2024-04-22 21:22:34.021862703 +0000 UTC m=+0.640776139 container remove 878d76986ae8f5d27124c0276ca0fa6636a69e8699f3a5fa9655ca8d8cfda59f (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, name=centos-stream, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, ceph=True, maintainer=Guillaume Abrioux , version=8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870) 2024-04-22T21:22:34.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-22T21:22:34.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-04-22T21:22:34.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 systemd[1]: Stopped Ceph mgr.y for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:22:34.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.y.service: Consumed 59.631s CPU time. 2024-04-22T21:22:34.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:34 smithi130 ceph-mon[21867]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:34.578 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 systemd[1]: Starting Ceph mgr.y for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:22:34.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:34 smithi130 ceph-mon[26394]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:34.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:34 smithi150 ceph-mon[22676]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:34.853 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 podman[60085]: 2024-04-22 21:22:34.495694657 +0000 UTC m=+0.018082510 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:22:34.853 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 podman[60085]: 2024-04-22 21:22:34.615128178 +0000 UTC m=+0.137516022 container create 2854a3fbb395345090e04b7d6a0c5e98baceef935557417cf9744b695541bc12 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y, RELEASE=main-56e81df, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-22T21:22:34.853 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 podman[60085]: 2024-04-22 21:22:34.809491888 +0000 UTC m=+0.331879731 container init 2854a3fbb395345090e04b7d6a0c5e98baceef935557417cf9744b695541bc12 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, org.label-schema.build-date=20240416, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2) 2024-04-22T21:22:34.854 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 podman[60085]: 2024-04-22 21:22:34.811977497 +0000 UTC m=+0.334365338 container start 2854a3fbb395345090e04b7d6a0c5e98baceef935557417cf9744b695541bc12 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-22T21:22:34.854 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 ceph-mgr[60104]: -- 172.21.15.130:0/2832392366 <== mon.1 v2:172.21.15.130:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f6c24f05a0 con 0x55f6c24c5000 2024-04-22T21:22:34.854 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 bash[60085]: 2854a3fbb395345090e04b7d6a0c5e98baceef935557417cf9744b695541bc12 2024-04-22T21:22:35.149 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 systemd[1]: Started Ceph mgr.y for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:22:35.149 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:34.967+0000 7f63dc683200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:22:35.549 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:35.162+0000 7f63dc683200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:22:35.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:35.270+0000 7f63dc683200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:22:35.920 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:35.620+0000 7f63dc683200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:22:35.921 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:35.707+0000 7f63dc683200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:22:36.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:36.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:36.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:36.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:36.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:36.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:36.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:36.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:36.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:36.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:35 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:36.353 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:35 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:36.353 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:35 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:36.353 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:35 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:36.353 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:35 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:36.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:35 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:37.261 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:36 smithi130 ceph-mon[21867]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:37.262 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:36.992+0000 7f63dc683200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:22:37.262 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:37 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:37.170+0000 7f63dc683200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:22:37.262 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:36 smithi130 ceph-mon[26394]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:37.353 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:36 smithi150 ceph-mon[22676]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:37.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:37 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:37.353+0000 7f63dc683200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:22:37.948 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:37 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:37.659+0000 7f63dc683200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:22:37.948 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:37 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:37.753+0000 7f63dc683200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:22:38.252 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:37 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:37.962+0000 7f63dc683200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:22:38.252 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:38.157+0000 7f63dc683200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:22:38.853 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:22:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:22:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:22:39.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:38.763+0000 7f63dc683200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:22:39.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:38 smithi130 ceph-mon[26394]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:39.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:38 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:22:39.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:38 smithi130 ceph-mon[21867]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:39.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:38 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:22:39.219 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:38 smithi150 ceph-mon[22676]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:39.220 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:38 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:22:39.491 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:39.200+0000 7f63dc683200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:22:39.491 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:39.299+0000 7f63dc683200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:22:39.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:39.505+0000 7f63dc683200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T21:22:39.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:22:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:22:39] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-04-22T21:22:40.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:40.039+0000 7f63dc683200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:22:40.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:40.134+0000 7f63dc683200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:22:40.729 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:40.385+0000 7f63dc683200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:22:40.729 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:40.522+0000 7f63dc683200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:22:40.729 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:40.622+0000 7f63dc683200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:22:40.729 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:40 smithi130 ceph-mon[26394]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:40.729 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:40 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:40.730 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:40 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:40.730 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:40 smithi130 ceph-mon[21867]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:40.730 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:40 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:40.730 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:40 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:40.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:40 smithi150 ceph-mon[22676]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:40.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:40 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:40.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:40 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:41.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:40.742+0000 7f63dc683200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:22:41.549 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:41 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:41.063+0000 7f63dc683200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:22:42.000 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:41 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:41.685+0000 7f63dc683200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:22:42.001 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:41 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:41.906+0000 7f63dc683200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:22:42.001 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:41 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:22:41] ENGINE Bus STARTING 2024-04-22T21:22:42.001 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CherryPy Checker: 2024-04-22T21:22:42.001 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: The Application mounted at '' has an empty config. 2024-04-22T21:22:42.001 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:22:42.001 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:22:42] ENGINE Serving on http://:::9283 2024-04-22T21:22:42.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:22:42] ENGINE Bus STARTED 2024-04-22T21:22:42.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[26394]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:42.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[26394]: Standby manager daemon y restarted 2024-04-22T21:22:42.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[26394]: Standby manager daemon y started 2024-04-22T21:22:42.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T21:22:42.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:22:42.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T21:22:42.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:22:42.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[21867]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:42.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[21867]: Standby manager daemon y restarted 2024-04-22T21:22:42.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[21867]: Standby manager daemon y started 2024-04-22T21:22:42.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T21:22:42.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:22:42.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T21:22:42.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:42 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:22:42.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:42 smithi150 ceph-mon[22676]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:42.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:42 smithi150 ceph-mon[22676]: Standby manager daemon y restarted 2024-04-22T21:22:42.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:42 smithi150 ceph-mon[22676]: Standby manager daemon y started 2024-04-22T21:22:42.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:42 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T21:22:42.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:42 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:22:42.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:42 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T21:22:42.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:42 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:22:43.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:43 smithi130 ceph-mon[21867]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-22T21:22:43.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:43 smithi130 ceph-mon[26394]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-22T21:22:43.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:43 smithi150 ceph-mon[22676]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-22T21:22:43.854 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:22:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:22:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:22:44.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:44 smithi130 ceph-mon[21867]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:44.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:44 smithi130 ceph-mon[26394]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:44.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:44 smithi150 ceph-mon[22676]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:45.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:45 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:45.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:45 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:45.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:45 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:46.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:46 smithi130 ceph-mon[21867]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:46.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:46 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:46.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:46 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:46.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:46 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:22:46.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:46 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:22:46.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:46 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:46.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:46 smithi130 ceph-mon[26394]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:46.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:46 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:46.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:46 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:46.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:46 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:22:46.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:46 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:22:46.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:46 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:46.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:46 smithi150 ceph-mon[22676]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:46.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:46 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:46.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:46 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:46.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:46 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:22:46.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:46 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:22:46.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:46 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:22:48.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:48 smithi130 ceph-mon[26394]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:48.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:48 smithi130 ceph-mon[21867]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:48.853 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:22:48 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:22:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:22:48.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:48 smithi150 ceph-mon[22676]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:49.816 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:22:49 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:22:49] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-04-22T21:22:50.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:50 smithi130 ceph-mon[21867]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:50.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:50 smithi130 ceph-mon[26394]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:50 smithi150 ceph-mon[22676]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:52.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:52 smithi150 ceph-mon[22676]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:52.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:52 smithi130 ceph-mon[26394]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:52.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:52 smithi130 ceph-mon[21867]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:53.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:53 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:22:53.604 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:22:53 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:22:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:22:53.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:53 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:22:53.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:53 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:22:54.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:54 smithi130 ceph-mon[21867]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:54.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:54 smithi130 ceph-mon[26394]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:54.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:54 smithi150 ceph-mon[22676]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:55.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:55 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:55.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:55 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:55.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:55 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:22:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:56 smithi150 ceph-mon[22676]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:56.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:56 smithi130 ceph-mon[21867]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:56.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:56 smithi130 ceph-mon[26394]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:22:58.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:22:58 smithi150 ceph-mon[22676]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:58.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:22:58 smithi130 ceph-mon[21867]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:58.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:22:58 smithi130 ceph-mon[26394]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:22:59.103 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:22:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:22:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:22:59.603 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:22:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:22:59] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-04-22T21:23:00.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:00 smithi130 ceph-mon[21867]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:00.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:00 smithi130 ceph-mon[26394]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:00.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:00 smithi150 ceph-mon[22676]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:02.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:02 smithi130 ceph-mon[21867]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:02.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:02 smithi130 ceph-mon[26394]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:02.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:02 smithi150 ceph-mon[22676]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:03.853 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:23:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:23:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:23:04.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:04 smithi130 ceph-mon[21867]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:04.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:04 smithi130 ceph-mon[26394]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:04.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:04 smithi150 ceph-mon[22676]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:05.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:05 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:05.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:05 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:05.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:05 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:06.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:06 smithi150 ceph-mon[22676]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:06.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:06 smithi130 ceph-mon[21867]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:06.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:06 smithi130 ceph-mon[26394]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:08 smithi150 ceph-mon[22676]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:08 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:23:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:08 smithi130 ceph-mon[26394]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:08 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:23:08.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:08 smithi130 ceph-mon[21867]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:08.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:08 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:23:09.103 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:23:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:23:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:23:09.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:23:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:23:09] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-04-22T21:23:10.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:10 smithi130 ceph-mon[21867]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:10.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:10 smithi130 ceph-mon[26394]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:10.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:10 smithi150 ceph-mon[22676]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:12.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:12 smithi150 ceph-mon[22676]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:12.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:12 smithi130 ceph-mon[26394]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:12.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:12 smithi130 ceph-mon[21867]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:13.853 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:23:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:23:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:23:14.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:14 smithi130 ceph-mon[26394]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:14.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:14 smithi130 ceph-mon[21867]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:14.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:14 smithi150 ceph-mon[22676]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:15.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:15 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:15.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:15 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:15.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:15 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:16.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:16 smithi150 ceph-mon[22676]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:16.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:16 smithi130 ceph-mon[26394]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:16.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:16 smithi130 ceph-mon[21867]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:18 smithi150 ceph-mon[22676]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:18.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:18 smithi130 ceph-mon[26394]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:18 smithi130 ceph-mon[21867]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:19.103 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:23:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:23:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:23:19.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:23:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:23:19] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-04-22T21:23:20.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:20 smithi130 ceph-mon[26394]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:20.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:20 smithi130 ceph-mon[21867]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:20.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:20 smithi150 ceph-mon[22676]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:22.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:22 smithi150 ceph-mon[22676]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:22.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:22 smithi130 ceph-mon[21867]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:22.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:22 smithi130 ceph-mon[26394]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:23.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:23 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:23:23.603 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:23:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:23:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:23:23.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:23 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:23:23.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:23 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:23:24.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:24 smithi130 ceph-mon[21867]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:24.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:24 smithi130 ceph-mon[26394]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:24.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:24 smithi150 ceph-mon[22676]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:25.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:25 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:25.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:25 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:25.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:25 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:26.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:26 smithi150 ceph-mon[22676]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:26.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:26 smithi130 ceph-mon[21867]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:26.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:26 smithi130 ceph-mon[26394]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:28.645 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:28 smithi150 ceph-mon[22676]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:28.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:28 smithi130 ceph-mon[26394]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:28.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:28 smithi130 ceph-mon[21867]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:29.103 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:23:28 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:23:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:23:29.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:23:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:23:29] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-04-22T21:23:30.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:30 smithi130 ceph-mon[26394]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:30.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:30 smithi130 ceph-mon[21867]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:30.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:30 smithi150 ceph-mon[22676]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:32.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:32 smithi150 ceph-mon[22676]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:32.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:32 smithi130 ceph-mon[26394]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:32.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:32 smithi130 ceph-mon[21867]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:33.853 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:23:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:23:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:23:34.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:34 smithi130 ceph-mon[21867]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:34.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:34 smithi130 ceph-mon[26394]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:34.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:34 smithi150 ceph-mon[22676]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:35.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:35 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:35.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:35 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:35.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:35 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:36.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:36 smithi150 ceph-mon[22676]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:36.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:36 smithi130 ceph-mon[26394]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:36.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:36 smithi130 ceph-mon[21867]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:38.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:38 smithi150 ceph-mon[22676]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:38.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:38 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:23:38.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:38 smithi130 ceph-mon[21867]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:38.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:38 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:23:38.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:38 smithi130 ceph-mon[26394]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:38.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:38 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:23:39.103 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:23:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:23:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:23:39.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:23:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:23:39] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-04-22T21:23:40.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:40 smithi130 ceph-mon[21867]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:40.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:40 smithi130 ceph-mon[26394]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:40.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:40 smithi150 ceph-mon[22676]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:42.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:42 smithi150 ceph-mon[22676]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:42.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:42 smithi130 ceph-mon[21867]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:42.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:42 smithi130 ceph-mon[26394]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:43.853 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:23:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:23:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:23:44.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:44 smithi130 ceph-mon[21867]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:44.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:44 smithi130 ceph-mon[26394]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:44.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:44 smithi150 ceph-mon[22676]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:45 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:45.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:45 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:45.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:45 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:46.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:46 smithi150 ceph-mon[22676]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:46.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:46 smithi130 ceph-mon[21867]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:46.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:46 smithi130 ceph-mon[26394]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:48.448 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:48 smithi130 ceph-mon[26394]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:48.448 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:48 smithi130 ceph-mon[21867]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:48.644 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:48 smithi150 ceph-mon[22676]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:49.103 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:23:48 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:23:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:23:49.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:23:49 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:23:49] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-04-22T21:23:50.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:50 smithi130 ceph-mon[26394]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:50.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:50 smithi130 ceph-mon[21867]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:50 smithi150 ceph-mon[22676]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:52.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:52 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:23:52.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:52 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:23:52.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:52 smithi130 ceph-mon[21867]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:52.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:52 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:23:52.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:52 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:23:52.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:52 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:23:52.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:52 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:23:52.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:52 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:23:52.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:52 smithi130 ceph-mon[26394]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:52.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:52 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:23:52.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:52 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:23:52.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:52 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:23:52.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:52 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:23:52.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:52 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:23:52.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:52 smithi150 ceph-mon[22676]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:52.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:52 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:23:52.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:52 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:23:52.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:52 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:23:53.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:53 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:23:53.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:53 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:23:53.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:53 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:23:53.853 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:23:53 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:23:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:23:54.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:54 smithi130 ceph-mon[21867]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:54.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:54 smithi130 ceph-mon[26394]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:54 smithi150 ceph-mon[22676]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:55.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:55 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:55.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:55 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:55.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:55 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:23:56.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:56 smithi130 ceph-mon[21867]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:56.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:56 smithi130 ceph-mon[26394]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:56.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:56 smithi150 ceph-mon[22676]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:23:58.645 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:23:58 smithi150 ceph-mon[22676]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:58.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:23:58 smithi130 ceph-mon[26394]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:58.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:23:58 smithi130 ceph-mon[21867]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:23:59.103 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:23:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:23:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:23:59.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:23:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:23:59] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-04-22T21:24:00.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:00 smithi130 ceph-mon[21867]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:00.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:00 smithi130 ceph-mon[26394]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:00.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:00 smithi150 ceph-mon[22676]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:02.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:02 smithi150 ceph-mon[22676]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:02.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:02 smithi130 ceph-mon[21867]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:02.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:02 smithi130 ceph-mon[26394]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:03.853 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:24:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:24:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:24:04.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:04 smithi130 ceph-mon[21867]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:04.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:04 smithi130 ceph-mon[26394]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:04.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:04 smithi150 ceph-mon[22676]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:05.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:05 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:05.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:05 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:05.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:05 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:06.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:06 smithi130 ceph-mon[21867]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:06.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:06 smithi130 ceph-mon[26394]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:06.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:06 smithi150 ceph-mon[22676]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:08.644 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:08 smithi150 ceph-mon[22676]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:08.644 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:08 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:24:08.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:08 smithi130 ceph-mon[21867]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:08.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:08 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:24:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:08 smithi130 ceph-mon[26394]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:08 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:24:09.103 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:24:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:24:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"172.21.15.150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:24:09.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:24:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:24:09] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-04-22T21:24:10.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:10 smithi130 ceph-mon[21867]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:10.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:10 smithi130 ceph-mon[26394]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:10.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:10 smithi150 ceph-mon[22676]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:12.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:12 smithi150 ceph-mon[22676]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:12.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:12 smithi130 ceph-mon[26394]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:12.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:12 smithi130 ceph-mon[21867]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:13.853 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:24:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:24:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi150\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi150\", job=\"node\", machine=\"x86_64\", nodename=\"smithi150\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"dd7d46b8-00eb-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi150\", job=\"node\", machine=\"x86_64\", nodename=\"smithi150\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T21:24:14.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:14 smithi130 ceph-mon[26394]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:14.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:14 smithi130 ceph-mon[21867]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:14.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:14 smithi150 ceph-mon[22676]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:15.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:15 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:15.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:15 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:15.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:15 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:16.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:16 smithi130 ceph-mon[26394]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:16.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:16 smithi130 ceph-mon[21867]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:16.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:16 smithi150 ceph-mon[22676]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:18.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:18 smithi150 ceph-mon[22676]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:18.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:18 smithi130 ceph-mon[26394]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:18.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:18 smithi130 ceph-mon[21867]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:19.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:24:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:24:19] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-04-22T21:24:20.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:20 smithi130 ceph-mon[26394]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:20.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:20 smithi130 ceph-mon[21867]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:20.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:20 smithi150 ceph-mon[22676]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:22.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:22 smithi130 ceph-mon[26394]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:22.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:22 smithi130 ceph-mon[21867]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:22.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:22 smithi150 ceph-mon[22676]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:23.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:23 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:24:23.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:23 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:24:23.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:23 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:24:24.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:24 smithi130 ceph-mon[21867]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:24.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:24 smithi130 ceph-mon[26394]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:24.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:24 smithi150 ceph-mon[22676]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:25.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:25 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:25.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:25 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:25 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:26.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:26 smithi130 ceph-mon[21867]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:26.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:26 smithi130 ceph-mon[26394]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:26.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:26 smithi150 ceph-mon[22676]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:28.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:28 smithi130 ceph-mon[21867]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:28.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:28 smithi130 ceph-mon[26394]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:28.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:28 smithi150 ceph-mon[22676]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:29.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:24:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:24:29] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-04-22T21:24:30.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:30 smithi130 ceph-mon[21867]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:30.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:30 smithi130 ceph-mon[26394]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:30.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:30 smithi150 ceph-mon[22676]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:32.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:32 smithi130 ceph-mon[21867]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:32.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:32 smithi130 ceph-mon[26394]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:32.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:32 smithi150 ceph-mon[22676]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:34.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:34 smithi130 ceph-mon[21867]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:34.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:34 smithi130 ceph-mon[26394]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:34.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:34 smithi150 ceph-mon[22676]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:35.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:35 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:35.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:35 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:35.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:35 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:36.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:36 smithi130 ceph-mon[21867]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:36.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:36 smithi130 ceph-mon[26394]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:36.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:36 smithi150 ceph-mon[22676]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:38.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:38 smithi130 ceph-mon[21867]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:38.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:38 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:24:38.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:38 smithi130 ceph-mon[26394]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:38.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:38 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:24:38.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:38 smithi150 ceph-mon[22676]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:38.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:38 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:24:39.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:24:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:24:39] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-04-22T21:24:40.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:40 smithi130 ceph-mon[21867]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:40.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:40 smithi130 ceph-mon[26394]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:40.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:40 smithi150 ceph-mon[22676]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:42.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:42 smithi130 ceph-mon[26394]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:42.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:42 smithi130 ceph-mon[21867]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:42.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:42 smithi150 ceph-mon[22676]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:44.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:44 smithi130 ceph-mon[26394]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:44.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:44 smithi130 ceph-mon[21867]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:44.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:44 smithi150 ceph-mon[22676]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:45.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:45 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:45.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:45 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:45.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:45 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:46.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:46 smithi130 ceph-mon[21867]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:46.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:46 smithi130 ceph-mon[26394]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:46 smithi150 ceph-mon[22676]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:48 smithi150 ceph-mon[22676]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:48.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:48 smithi130 ceph-mon[26394]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:48.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:48 smithi130 ceph-mon[21867]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:49.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:24:49 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:24:49] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-04-22T21:24:50.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:50 smithi130 ceph-mon[21867]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:50.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:50 smithi130 ceph-mon[26394]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:50.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:50 smithi150 ceph-mon[22676]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:52.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:52 smithi130 ceph-mon[21867]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:52.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:52 smithi130 ceph-mon[26394]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:52.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:52 smithi150 ceph-mon[22676]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:53.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:53 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:24:53.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:53 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:24:53.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:53 smithi130 ceph-mon[21867]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:24:53.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:53 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:24:53.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:53 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:24:53.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:53 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:24:53.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:53 smithi130 ceph-mon[26394]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:24:53.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:53 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:24:53.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:53 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:24:53.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:53 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:24:53.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:53 smithi150 ceph-mon[22676]: from='mgr.34760 ' entity='mgr.x' 2024-04-22T21:24:53.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:53 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:24:54.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:54 smithi130 ceph-mon[21867]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:54 smithi130 ceph-mon[26394]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:54.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:54 smithi150 ceph-mon[22676]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:55.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:55 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:55.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:55 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:55.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:55 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:24:56.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:56 smithi130 ceph-mon[26394]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:56.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:56 smithi130 ceph-mon[21867]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:56.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:56 smithi150 ceph-mon[22676]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:24:58.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:24:58 smithi130 ceph-mon[26394]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:58.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:24:58 smithi130 ceph-mon[21867]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:58.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:24:58 smithi150 ceph-mon[22676]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:24:59.852 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:24:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:24:59] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-04-22T21:25:00.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:00 smithi130 ceph-mon[26394]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:25:00.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:00 smithi130 ceph-mon[21867]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:25:00.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:00 smithi150 ceph-mon[22676]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:25:02.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:02 smithi130 ceph-mon[26394]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:02.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:02 smithi130 ceph-mon[21867]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:02.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:02 smithi150 ceph-mon[22676]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:04.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:04 smithi130 ceph-mon[21867]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:04.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:04 smithi130 ceph-mon[26394]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:04 smithi150 ceph-mon[22676]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:04.941 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps' 2024-04-22T21:25:05.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:05 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:25:05.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:05 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:25:05.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:05 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:25:06.453 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:06 smithi130 ceph-mon[21867]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:25:06.453 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:06 smithi130 ceph-mon[26394]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:25:06.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:06 smithi150 ceph-mon[22676]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:25:06.954 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:25:06.954 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (5m) 75s ago 13m 24.6M - 0.23.0 ba2b418f427c 23fd04f877ce 2024-04-22T21:25:06.954 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (5m) 2m ago 13m 41.5M - 8.3.5 dad864ee21e9 e021384d6729 2024-04-22T21:25:06.954 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (4m) 75s ago 12m 55.2M - 3.5 e1d6a67b021e 0a30d8841db2 2024-04-22T21:25:06.954 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283 running (9m) 2m ago 18m 510M - 19.0.0-3198-g56e81df3 82086b19bb07 327605fe868f 2024-04-22T21:25:06.954 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (2m) 75s ago 20m 448M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:25:06.954 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (20m) 75s ago 20m 59.0M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:25:06.954 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (18m) 2m ago 18m 43.5M 2048M 17.2.0 e1d6a67b021e 861909a68cf9 2024-04-22T21:25:06.954 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (18m) 75s ago 18m 44.0M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:25:06.954 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (6m) 75s ago 14m 15.1M - 1.3.1 1dbe0e931976 4057e105dc28 2024-04-22T21:25:06.954 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (5m) 2m ago 14m 17.3M - 1.3.1 1dbe0e931976 2bcf91466257 2024-04-22T21:25:06.955 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (18m) 75s ago 18m 53.4M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:25:06.955 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (17m) 75s ago 17m 55.8M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:25:06.955 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (17m) 75s ago 17m 49.6M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:25:06.955 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (16m) 75s ago 16m 52.0M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:25:06.955 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (16m) 2m ago 16m 52.0M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:25:06.955 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (16m) 2m ago 16m 51.1M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:25:06.955 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (15m) 2m ago 15m 52.8M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:25:06.955 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (15m) 2m ago 15m 52.1M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:25:06.955 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (5m) 2m ago 14m 70.5M - 2.33.4 514e6a882f6e 686c0cd09608 2024-04-22T21:25:06.955 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (13m) 75s ago 13m 74.8M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:25:06.955 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (13m) 2m ago 13m 71.6M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:25:07.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:07 smithi130 ceph-mon[21867]: from='client.25120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:25:07.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:07 smithi130 ceph-mon[26394]: from='client.25120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:25:07.576 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions' 2024-04-22T21:25:07.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:07 smithi150 ceph-mon[22676]: from='client.25120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:25:08.742 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:08 smithi130 ceph-mon[21867]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:08.743 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:08 smithi130 ceph-mon[21867]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:25:08.743 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:08 smithi130 ceph-mon[26394]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:08.743 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:08 smithi130 ceph-mon[26394]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:25:08.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:08 smithi150 ceph-mon[22676]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:08.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:08 smithi150 ceph-mon[22676]: from='mgr.34760 172.21.15.150:0/1177129422' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:25:09.701 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:25:09.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:25:09.701 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T21:25:09.701 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:25:09.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:25:09.701 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:25:09.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:25:09.702 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:25:09.702 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:25:09.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:25:09.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": {}, 2024-04-22T21:25:09.702 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:25:09.702 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:25:09.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:25:09.702 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:25:09.702 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T21:25:09.702 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:25:09.702 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:25:09.702 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:25:09.852 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:25:09] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-04-22T21:25:10.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:10 smithi130 ceph-mon[26394]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:25:10.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:10 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3761589889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:25:10.299 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:10 smithi130 ceph-mon[21867]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:25:10.299 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:10 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3761589889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:25:10.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph health detail' 2024-04-22T21:25:10.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:10 smithi150 ceph-mon[22676]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:25:10.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:10 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3761589889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:25:12.545 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-04-22T21:25:12.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:12 smithi130 ceph-mon[21867]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:12.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:12 smithi130 ceph-mon[26394]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:12.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:12 smithi150 ceph-mon[22676]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:13.174 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph -s' 2024-04-22T21:25:13.660 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:13 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1295764319' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:25:13.661 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:13 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1295764319' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:25:13.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:13 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1295764319' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:25:14.700 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:14 smithi130 ceph-mon[21867]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:14.701 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:14 smithi130 ceph-mon[26394]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:14.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:14 smithi150 ceph-mon[22676]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:15.221 INFO:teuthology.orchestra.run.smithi130.stdout: cluster: 2024-04-22T21:25:15.221 INFO:teuthology.orchestra.run.smithi130.stdout: id: dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:25:15.221 INFO:teuthology.orchestra.run.smithi130.stdout: health: HEALTH_OK 2024-04-22T21:25:15.221 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:25:15.221 INFO:teuthology.orchestra.run.smithi130.stdout: services: 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: mgr: x(active, since 6m), standbys: y 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: data: 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: pools: 6 pools, 161 pgs 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: objects: 209 objects, 585 KiB 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: pgs: 161 active+clean 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: io: 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-22T21:25:15.222 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:25:15.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:15 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:25:15.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:15 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3072339068' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:25:15.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:15 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:25:15.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:15 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3072339068' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:25:15.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:15 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:25:15.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:15 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3072339068' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:25:15.924 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph mgr fail' 2024-04-22T21:25:16.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:16 smithi130 ceph-mon[26394]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:25:16.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:16 smithi130 ceph-mon[21867]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:25:16.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:16 smithi150 ceph-mon[22676]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:25:18.487 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:18 smithi130 ceph-mon[26394]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:18.487 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:18 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3011143481' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T21:25:18.487 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:18 smithi130 ceph-mon[26394]: osdmap e104: 8 total, 8 up, 8 in 2024-04-22T21:25:18.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:25:18] ENGINE Bus STOPPING 2024-04-22T21:25:18.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:25:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T21:25:18.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:25:18] ENGINE Bus STOPPED 2024-04-22T21:25:18.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:18 smithi130 ceph-mon[21867]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:18.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:18 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3011143481' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T21:25:18.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:18 smithi130 ceph-mon[21867]: osdmap e104: 8 total, 8 up, 8 in 2024-04-22T21:25:18.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:18 smithi150 ceph-mon[22676]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:25:18.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:18 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3011143481' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T21:25:18.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:18 smithi150 ceph-mon[22676]: osdmap e104: 8 total, 8 up, 8 in 2024-04-22T21:25:18.656 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:18.412+0000 7f4097461700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-22T21:25:18.656 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ignoring --setuser ceph since I am not root 2024-04-22T21:25:18.656 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ignoring --setgroup ceph since I am not root 2024-04-22T21:25:18.656 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:18 smithi150 ceph-mgr[42406]: -- 172.21.15.150:0/3664237779 <== mon.2 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e39e5905a0 con 0x55e39e565000 2024-04-22T21:25:18.656 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:18.592+0000 7fbf6d8ef200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:25:18.789 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:25:18] ENGINE Bus STARTING 2024-04-22T21:25:18.789 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:25:18] ENGINE Serving on http://:::9283 2024-04-22T21:25:18.789 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:25:18] ENGINE Bus STARTED 2024-04-22T21:25:19.102 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:18.781+0000 7fbf6d8ef200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:25:19.102 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:18.890+0000 7fbf6d8ef200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:25:19.180 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'sleep 180' 2024-04-22T21:25:19.399 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3011143481' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T21:25:19.399 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: mgrmap e28: y(active, starting, since 0.434748s) 2024-04-22T21:25:19.399 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:25:19.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:25:19.401 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: Manager daemon y is now available 2024-04-22T21:25:19.401 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:25:19.401 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:25:19.401 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:25:19.401 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3011143481' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T21:25:19.401 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: mgrmap e28: y(active, starting, since 0.434748s) 2024-04-22T21:25:19.401 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:25:19.401 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:25:19.401 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:25:19.401 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:25:19.401 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:25:19.401 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:25:19.401 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:25:19.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:25:19.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:25:19.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:25:19.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:25:19.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:25:19.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:25:19.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:25:19.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:25:19.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: Manager daemon y is now available 2024-04-22T21:25:19.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:25:19.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:25:19.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:25:19.577 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:19.218+0000 7fbf6d8ef200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:25:19.577 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:19.316+0000 7fbf6d8ef200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:25:19.577 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3011143481' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T21:25:19.577 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: mgrmap e28: y(active, starting, since 0.434748s) 2024-04-22T21:25:19.577 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:25:19.577 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:25:19.578 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:25:19.578 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:25:19.578 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:25:19.578 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:25:19.578 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:25:19.578 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:25:19.578 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:25:19.578 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:25:19.578 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:25:19.578 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:25:19.578 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:25:19.578 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:25:19.578 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:25:19.579 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: Manager daemon y is now available 2024-04-22T21:25:19.579 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:25:19.579 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:25:19.579 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:25:20.729 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:20.570+0000 7fbf6d8ef200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:25:20.729 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-mon[22676]: [22/Apr/2024:21:25:18] ENGINE Bus STARTING 2024-04-22T21:25:20.729 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-mon[22676]: [22/Apr/2024:21:25:19] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:25:20.729 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-mon[22676]: [22/Apr/2024:21:25:19] ENGINE Serving on http://172.21.15.130:8765 2024-04-22T21:25:20.729 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-mon[22676]: [22/Apr/2024:21:25:19] ENGINE Bus STARTED 2024-04-22T21:25:20.729 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-mon[22676]: mgrmap e29: y(active, since 1.44534s) 2024-04-22T21:25:20.729 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-mon[22676]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:20.730 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:20.560Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.150:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.150:8765: connect: connection refused" 2024-04-22T21:25:20.730 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:20.560Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.150:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.150:8765: connect: connection refused" 2024-04-22T21:25:20.730 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:20.560Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.150:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.150:8765: connect: connection refused" 2024-04-22T21:25:20.730 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:20.563Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.150:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.150:8765: connect: connection refused" 2024-04-22T21:25:20.730 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:20.563Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.150:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.150:8765: connect: connection refused" 2024-04-22T21:25:20.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:20 smithi130 ceph-mon[21867]: [22/Apr/2024:21:25:18] ENGINE Bus STARTING 2024-04-22T21:25:20.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:20 smithi130 ceph-mon[21867]: [22/Apr/2024:21:25:19] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:25:20.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:20 smithi130 ceph-mon[21867]: [22/Apr/2024:21:25:19] ENGINE Serving on http://172.21.15.130:8765 2024-04-22T21:25:20.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:20 smithi130 ceph-mon[21867]: [22/Apr/2024:21:25:19] ENGINE Bus STARTED 2024-04-22T21:25:20.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:20 smithi130 ceph-mon[21867]: mgrmap e29: y(active, since 1.44534s) 2024-04-22T21:25:20.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:20 smithi130 ceph-mon[21867]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:20.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:20 smithi130 ceph-mon[26394]: [22/Apr/2024:21:25:18] ENGINE Bus STARTING 2024-04-22T21:25:20.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:20 smithi130 ceph-mon[26394]: [22/Apr/2024:21:25:19] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:25:20.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:20 smithi130 ceph-mon[26394]: [22/Apr/2024:21:25:19] ENGINE Serving on http://172.21.15.130:8765 2024-04-22T21:25:20.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:20 smithi130 ceph-mon[26394]: [22/Apr/2024:21:25:19] ENGINE Bus STARTED 2024-04-22T21:25:20.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:20 smithi130 ceph-mon[26394]: mgrmap e29: y(active, since 1.44534s) 2024-04-22T21:25:20.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:20 smithi130 ceph-mon[26394]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:21.102 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:20.742+0000 7fbf6d8ef200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:25:21.152 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:20.930+0000 7fbf6d8ef200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:25:21.516 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:21 smithi150 ceph-mon[22676]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:21.516 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:21 smithi150 ceph-mon[22676]: mgrmap e30: y(active, since 2s) 2024-04-22T21:25:21.516 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:21 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:21.234+0000 7fbf6d8ef200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:25:21.516 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:21 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:21.341+0000 7fbf6d8ef200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:25:21.649 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:21 smithi130 ceph-mon[21867]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:21.649 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:21 smithi130 ceph-mon[21867]: mgrmap e30: y(active, since 2s) 2024-04-22T21:25:21.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:21 smithi130 ceph-mon[26394]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:21.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:21 smithi130 ceph-mon[26394]: mgrmap e30: y(active, since 2s) 2024-04-22T21:25:21.852 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:21 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:21.550+0000 7fbf6d8ef200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:25:21.852 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:21 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:21.751+0000 7fbf6d8ef200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:25:22.602 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:22.364+0000 7fbf6d8ef200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:25:23.102 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:22.815+0000 7fbf6d8ef200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:25:23.102 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:22.915+0000 7fbf6d8ef200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:25:23.420 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:23.125+0000 7fbf6d8ef200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T21:25:23.681 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:23 smithi150 ceph-mon[22676]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:23.681 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:23 smithi150 ceph-mon[22676]: mgrmap e31: y(active, since 4s) 2024-04-22T21:25:23.681 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:23 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:23.681 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:23 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:23.691 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:23 smithi130 ceph-mon[26394]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:23.692 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:23 smithi130 ceph-mon[26394]: mgrmap e31: y(active, since 4s) 2024-04-22T21:25:23.692 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:23 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:23.692 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:23 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:23.692 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:23 smithi130 ceph-mon[21867]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:23.693 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:23 smithi130 ceph-mon[21867]: mgrmap e31: y(active, since 4s) 2024-04-22T21:25:23.693 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:23 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:23.693 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:23 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:24.041 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:23.694+0000 7fbf6d8ef200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:25:24.041 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:23.793+0000 7fbf6d8ef200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:25:24.352 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:24.055+0000 7fbf6d8ef200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:25:24.352 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:24.199+0000 7fbf6d8ef200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:25:24.352 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:24.304+0000 7fbf6d8ef200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:25:24.721 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:24.426+0000 7fbf6d8ef200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:25:25.102 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:24.734+0000 7fbf6d8ef200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:25:25.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-mon[22676]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:25:25.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:25.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:25.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-mon[22676]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:25.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:25.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:25.603 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:25.344+0000 7fbf6d8ef200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:25:25.603 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:25.567+0000 7fbf6d8ef200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:25:25.603 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:25:25] ENGINE Bus STARTING 2024-04-22T21:25:25.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:25 smithi130 ceph-mon[21867]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:25:25.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:25 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:25.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:25 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:25.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:25 smithi130 ceph-mon[21867]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:25.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:25 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:25.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:25 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:25.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:25 smithi130 ceph-mon[26394]: from='client.25081 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:25:25.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:25 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:25.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:25 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:25.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:25 smithi130 ceph-mon[26394]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:25.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:25 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:25.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:25 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:26.103 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: CherryPy Checker: 2024-04-22T21:25:26.103 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: The Application mounted at '' has an empty config. 2024-04-22T21:25:26.103 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:25:26.103 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:25:25] ENGINE Serving on http://:::9283 2024-04-22T21:25:26.103 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:25 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:25:25] ENGINE Bus STARTED 2024-04-22T21:25:26.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[26394]: Standby manager daemon x started 2024-04-22T21:25:26.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:25:26.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:25:26.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:25:26.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:25:26.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:26.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:26.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:25:26.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:25:26.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[21867]: Standby manager daemon x started 2024-04-22T21:25:26.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:25:26.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:25:26.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:25:26.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:25:26.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:26.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:26.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:25:26.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:26 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:25:26.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:26 smithi150 ceph-mon[22676]: Standby manager daemon x started 2024-04-22T21:25:26.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:26 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:25:26.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:26 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:25:26.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:26 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:25:26.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:26 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:25:26.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:26 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:26.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:26 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:26.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:26 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:25:26.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:26 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:25:27.701 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:25:27.701 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:25:27.701 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:25:27.701 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:25:27.701 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: mgrmap e32: y(active, since 8s), standbys: x 2024-04-22T21:25:27.701 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:25:27.701 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:25:27.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:25:27.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:25:27.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:25:27.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:25:27.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:25:27.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:25:27.702 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: mgrmap e32: y(active, since 8s), standbys: x 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.704 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:25:27.704 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:27 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:25:27.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:25:27.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:25:27.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:25:27.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:25:27.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: mgrmap e32: y(active, since 8s), standbys: x 2024-04-22T21:25:27.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:25:27.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:25:27.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:25:27.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:25:27.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:25:27.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:25:27.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:27.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:25:27.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:27 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:25:28.741 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:28 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:25:28.741 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:28 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:25:28.741 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:28 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:25:28.741 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:28 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:25:28.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:28 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:25:28.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:28 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:25:29.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:29 smithi130 ceph-mon[26394]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:25:29.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:29 smithi130 ceph-mon[21867]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:25:29.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:29 smithi150 ceph-mon[22676]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:25:29.853 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:25:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:25:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:25:30.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:25:30.675+0000 7f6362244700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:30.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:25:30.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:30.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:30.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:30.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:30.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:30.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:25:30.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:30.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:25:30.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:30.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:25:30.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:30.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:30.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:30.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:30.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:30.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:30.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:30.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:30.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:30.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:25:30.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:25:30.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:25:30.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:25:30.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:25:30.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:25:30.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:25:30.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:25:30.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return future.result(timeout) 2024-04-22T21:25:30.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:25:30.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.__get_result() 2024-04-22T21:25:30.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:25:30.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: raise self._exception 2024-04-22T21:25:30.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:25:30.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: use_current_daemon_image=reconfig, 2024-04-22T21:25:30.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:25:30.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:25:30.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:30.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:25:30.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:30.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:30.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:30.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:30.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:30.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:25:30.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:30.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:25:30.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:30.804 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:25:30.804 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:30.804 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:30.804 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:30.804 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:30.804 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:30.804 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:30.804 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:30.804 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:30.804 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:31.776 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:31 smithi150 ceph-mon[22676]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:25:31.777 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:31 smithi150 ceph-mon[22676]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:31.777 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:25:31.777 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:31.777 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:31.777 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:31.777 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:31.777 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:31.777 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:25:31.777 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:31.777 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:25:31.777 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:31.777 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:25:31.778 INFO:journalctl@ceph.mon.b.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:25:31.779 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:25:31.779 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:25:31.779 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:25:31.779 INFO:journalctl@ceph.mon.b.smithi150.stdout: return future.result(timeout) 2024-04-22T21:25:31.779 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:25:31.779 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.__get_result() 2024-04-22T21:25:31.779 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:25:31.779 INFO:journalctl@ceph.mon.b.smithi150.stdout: raise self._exception 2024-04-22T21:25:31.779 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:25:31.779 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:25:31.779 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:25:31.779 INFO:journalctl@ceph.mon.b.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:25:31.779 INFO:journalctl@ceph.mon.b.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:31.780 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:25:31.780 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:31.780 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:31.780 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:31.780 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:31.780 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:31.780 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:25:31.780 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:31.780 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:25:31.780 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:31.780 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:25:31.780 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:31.780 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:31.781 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:31.781 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:31.781 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:31.781 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:31.781 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:31.781 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:31.781 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:31.781 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:31 smithi150 ceph-mon[22676]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:25:31.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:31 smithi130 ceph-mon[21867]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:25:31.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:31 smithi130 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:31.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:25:31.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:31.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:31.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:31.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:31.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:31.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:25:31.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:31.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:25:31.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:31.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:25:31.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:31.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:31.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:31.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:31.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:31.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:31.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:31.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:31.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: return future.result(timeout) 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.__get_result() 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: raise self._exception 2024-04-22T21:25:31.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:25:31.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:31.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:25:31.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:31.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:31.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:31.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:31.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:31.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:31.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:31.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:31.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:31.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:31 smithi130 ceph-mon[21867]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:25:31.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:31 smithi130 ceph-mon[26394]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:25:31.804 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:31 smithi130 ceph-mon[26394]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:31.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:25:31.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:31.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:31.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:31.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:31.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:31.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:25:31.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:31.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:25:31.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:31.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:25:31.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:25:31.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: return future.result(timeout) 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.__get_result() 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: raise self._exception 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:31.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:31.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:31.808 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:31.808 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:31.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:31 smithi130 ceph-mon[26394]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:25:32.751 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:32 smithi150 ceph-mon[22676]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:25:32.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:32 smithi130 ceph-mon[26394]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:25:32.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:32 smithi130 ceph-mon[21867]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:25:33.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:33 smithi150 ceph-mon[22676]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:25:33.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:33 smithi130 ceph-mon[26394]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:25:33.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:33 smithi130 ceph-mon[21867]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:25:34.468 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 systemd[1]: Stopping Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:25:34.468 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 bash[51403]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T21:25:34.468 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:34.412Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T21:25:34.469 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:34.412Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T21:25:34.469 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:34.412Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T21:25:34.469 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:34.412Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T21:25:34.469 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:34.412Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T21:25:34.469 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:34.412Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T21:25:34.469 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:34.412Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T21:25:34.469 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:34.412Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T21:25:34.469 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:34.413Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T21:25:34.469 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:34.413Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T21:25:34.469 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:34.413Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T21:25:34.469 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[46045]: ts=2024-04-22T21:25:34.413Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T21:25:34.469 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 podman[51414]: 2024-04-22 21:25:34.422652026 +0000 UTC m=+0.114983353 container died 686c0cd096083f51c5256c6573e9769e45a875c3d6e013eff85973378a788e32 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:25:34.779 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:34 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:25:34.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:34 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:25:34.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:34 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:25:35.072 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 podman[51414]: 2024-04-22 21:25:34.795270043 +0000 UTC m=+0.487601369 container cleanup 686c0cd096083f51c5256c6573e9769e45a875c3d6e013eff85973378a788e32 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:25:35.073 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 bash[51414]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T21:25:35.073 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:34 smithi150 bash[51440]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T21:25:35.352 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 podman[51426]: 2024-04-22 21:25:35.088090813 +0000 UTC m=+0.667702822 container remove 686c0cd096083f51c5256c6573e9769e45a875c3d6e013eff85973378a788e32 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:25:35.353 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T21:25:35.353 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 systemd[1]: Stopped Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:25:35.353 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 2.008s CPU time. 2024-04-22T21:25:35.353 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 systemd[1]: Starting Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:25:35.353 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 podman[51512]: 2024-04-22 21:25:35.233287664 +0000 UTC m=+0.019751375 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T21:25:35.353 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 podman[51512]: 2024-04-22 21:25:35.330743769 +0000 UTC m=+0.117207468 container create 33d2ba18f8405b3b673795f007ade0c5a09cc6f5036e6eed1a1fd5b3de882359 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:25:35.737 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:35 smithi130 ceph-mon[21867]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:25:35.737 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:35 smithi130 ceph-mon[26394]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:25:35.786 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-mon[22676]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:25:35.787 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 podman[51512]: 2024-04-22 21:25:35.566350469 +0000 UTC m=+0.352814166 container init 33d2ba18f8405b3b673795f007ade0c5a09cc6f5036e6eed1a1fd5b3de882359 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:25:35.787 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 podman[51512]: 2024-04-22 21:25:35.568898632 +0000 UTC m=+0.355362331 container start 33d2ba18f8405b3b673795f007ade0c5a09cc6f5036e6eed1a1fd5b3de882359 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:25:35.787 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:35.592Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T21:25:35.787 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:35.592Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T21:25:35.787 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:35.592Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T21:25:35.787 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:35.592Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi150 (none))" 2024-04-22T21:25:35.787 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:35.592Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T21:25:35.787 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:35.592Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T21:25:35.787 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:35.593Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T21:25:35.787 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:35.594Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T21:25:35.788 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:35.594Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T21:25:35.788 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:35.597Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T21:25:35.788 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:35.597Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.502µs 2024-04-22T21:25:35.788 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:35.597Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T21:25:35.788 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 bash[51512]: 33d2ba18f8405b3b673795f007ade0c5a09cc6f5036e6eed1a1fd5b3de882359 2024-04-22T21:25:35.788 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 systemd[1]: Started Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:25:35.788 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:35.801Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-22T21:25:36.048 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:25:35] ENGINE Bus STOPPING 2024-04-22T21:25:36.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:25:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T21:25:36.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:25:35] ENGINE Bus STOPPED 2024-04-22T21:25:36.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:25:35] ENGINE Bus STARTING 2024-04-22T21:25:36.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:25:35] ENGINE Serving on http://:::9283 2024-04-22T21:25:36.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:25:35] ENGINE Bus STARTED 2024-04-22T21:25:36.102 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:36.049Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-22T21:25:36.580 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:36.254Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-22T21:25:36.580 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:36.255Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-22T21:25:36.580 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:36.255Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.838µs wal_replay_duration=657.818782ms total_replay_duration=657.863928ms 2024-04-22T21:25:36.580 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:36.261Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T21:25:36.580 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:36.261Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T21:25:36.580 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:36.261Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T21:25:36.581 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:36.282Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.720532ms db_storage=1.23µs remote_storage=1.138µs web_handler=299ns query_engine=745ns scrape=366.743µs scrape_sd=99.197µs notify=10.629µs notify_sd=23.349µs rules=19.868473ms 2024-04-22T21:25:36.581 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:25:36.282Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T21:25:36.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:36.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:36.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:25:36.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-mon[22676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:25:36.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T21:25:36.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-mon[22676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T21:25:36.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:36 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:37.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:37.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:37.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:25:37.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[26394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:25:37.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T21:25:37.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[26394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T21:25:37.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:37.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:37.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:37.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:25:37.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:25:37.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T21:25:37.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[21867]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T21:25:37.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:36 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:37.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:37 smithi150 ceph-mon[22676]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:25:38.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:37 smithi130 ceph-mon[26394]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:25:38.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:37 smithi130 ceph-mon[21867]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:25:40.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:39 smithi130 ceph-mon[26394]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:25:40.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:39 smithi130 ceph-mon[21867]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:25:40.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:39 smithi150 ceph-mon[22676]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:25:41.345 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:41.345 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:41.345 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:25:41.345 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:25:41.345 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:41.345 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[21867]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:25:41.345 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:25:41.345 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:25:41.346 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:25:41.346 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:25:41.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:41.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:41.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:25:41.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:25:41.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:41.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[26394]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:25:41.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:25:41.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:25:41.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:25:41.347 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:41 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:25:41.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:41 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:41.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:41 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:41.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:41 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:25:41.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:41 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:25:41.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:41 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:25:41.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:41 smithi150 ceph-mon[22676]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:25:41.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:41 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:25:41.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:41 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:25:41.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:41 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:25:41.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:41 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:25:42.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:42 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2890121446' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:25:42.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:42 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2890121446' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:25:42.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:42 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2890121446' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:25:43.760 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:43 smithi130 ceph-mon[21867]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:43.760 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:43 smithi130 ceph-mon[26394]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:43.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:43 smithi150 ceph-mon[22676]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:45.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:25:44.714+0000 7f6362244700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:45.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:25:45.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:45.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:45.050 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.050 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.050 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.050 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:25:45.050 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:45.050 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:25:45.050 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:45.050 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:25:45.050 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:45.050 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:45.050 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:45.050 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:45.051 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:45.051 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:45.051 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:45.051 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.051 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.051 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:25:45.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:25:45.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:25:45.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:25:45.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:25:45.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:25:45.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:25:45.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:25:45.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return future.result(timeout) 2024-04-22T21:25:45.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:25:45.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.__get_result() 2024-04-22T21:25:45.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:25:45.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: raise self._exception 2024-04-22T21:25:45.052 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: use_current_daemon_image=reconfig, 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:25:45.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:45.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:25:45.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:45.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:25:45.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:45.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:45.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:45.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:45.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:45.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:45.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:45.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.055 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:45 smithi130 ceph-mon[21867]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:45.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:45 smithi130 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:45.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:25:45.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:45.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: return future.result(timeout) 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:25:45.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.__get_result() 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: raise self._exception 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:45.552 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:45.553 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:45.554 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.554 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:45 smithi130 ceph-mon[26394]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:45.554 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:45 smithi130 ceph-mon[26394]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:45.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:25:45.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:45.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:45.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:45.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: return future.result(timeout) 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.__get_result() 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: raise self._exception 2024-04-22T21:25:45.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.558 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:45 smithi150 ceph-mon[22676]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:45 smithi150 ceph-mon[22676]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:25:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:25:45.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:25:45.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: return future.result(timeout) 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.__get_result() 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: raise self._exception 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:25:45.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi-foo-smithi130-pcsiph 2024-04-22T21:25:45.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: /usr/bin/podman: stderr Error: no such container ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:25:45.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:25:45.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:25:45.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:25:45.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:25:45.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:45.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:25:45.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:25:45.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:25:45.607 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:25:45.607 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:25:45.607 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:25:45.607 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:25:45.607 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:25:45.607 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:25:45.607 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:25:45.607 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:25:45.607 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:25:45.607 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:25:45.607 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:25:47.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:47 smithi130 ceph-mon[21867]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:47.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:47 smithi130 ceph-mon[26394]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:47.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:47 smithi150 ceph-mon[22676]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:49.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:25:48] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-04-22T21:25:49.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:49 smithi130 ceph-mon[26394]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:49.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:49 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:25:49.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:49 smithi130 ceph-mon[21867]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:49.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:49 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:25:49.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:49 smithi150 ceph-mon[22676]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:49.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:49 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:25:51.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:51 smithi130 ceph-mon[26394]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:51.837 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:51 smithi130 ceph-mon[21867]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:51.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:51 smithi150 ceph-mon[22676]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:53.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:53 smithi130 ceph-mon[26394]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:53.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:53 smithi130 ceph-mon[21867]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:53.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:53 smithi150 ceph-mon[22676]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:55.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:55 smithi130 ceph-mon[21867]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:55.302 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:55 smithi130 ceph-mon[26394]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:55.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:55 smithi150 ceph-mon[22676]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:56.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:56 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3845958343' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:25:56.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:56 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3845958343' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:25:56.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:56 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3845958343' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:25:57.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:57 smithi130 ceph-mon[26394]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:57.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:57 smithi130 ceph-mon[21867]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:57.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:57 smithi150 ceph-mon[22676]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:25:59.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:25:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:25:58] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-04-22T21:25:59.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:25:59 smithi130 ceph-mon[26394]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:59.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:25:59 smithi130 ceph-mon[21867]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:25:59.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:25:59 smithi150 ceph-mon[22676]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:26:01.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:01 smithi130 ceph-mon[26394]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:01.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:01 smithi130 ceph-mon[21867]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:01.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:01 smithi150 ceph-mon[22676]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:03.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:03 smithi130 ceph-mon[26394]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:03.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:03 smithi130 ceph-mon[21867]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:03.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:03 smithi150 ceph-mon[22676]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:04.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:04 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:26:04.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:04 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:26:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:04 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:26:05.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:05 smithi130 ceph-mon[21867]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:05.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:05 smithi130 ceph-mon[26394]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:05.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:05 smithi150 ceph-mon[22676]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:07.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:07 smithi130 ceph-mon[21867]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:07.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:07 smithi130 ceph-mon[21867]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:26:07.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:07 smithi130 ceph-mon[26394]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:07.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:07 smithi130 ceph-mon[26394]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:26:07.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:07 smithi150 ceph-mon[22676]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:07.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:07 smithi150 ceph-mon[22676]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:26:09.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:26:08] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.33.4" 2024-04-22T21:26:09.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:09 smithi130 ceph-mon[26394]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:26:09.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:09 smithi130 ceph-mon[21867]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:26:09.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:09 smithi150 ceph-mon[22676]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:26:11.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:11 smithi130 ceph-mon[26394]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:11.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:11 smithi130 ceph-mon[21867]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:11.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:11 smithi150 ceph-mon[22676]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:13.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:13 smithi130 ceph-mon[26394]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:13.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:13 smithi130 ceph-mon[21867]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:13.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:13 smithi150 ceph-mon[22676]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:15.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:15 smithi130 ceph-mon[26394]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:15.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:15 smithi130 ceph-mon[21867]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:15.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:15 smithi150 ceph-mon[22676]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:17.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:17 smithi130 ceph-mon[26394]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:17.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:17 smithi130 ceph-mon[26394]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:17.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:17 smithi130 ceph-mon[21867]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:17.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:17 smithi130 ceph-mon[21867]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:17.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:17 smithi150 ceph-mon[22676]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:17.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:17 smithi150 ceph-mon[22676]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:19.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:26:18] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-22T21:26:19.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:19 smithi130 ceph-mon[26394]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:19.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:19 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:26:19.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:19 smithi130 ceph-mon[21867]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:19.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:19 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:26:19.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:19 smithi150 ceph-mon[22676]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:19.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:19 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:26:21.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:21 smithi130 ceph-mon[21867]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:21.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:21 smithi130 ceph-mon[26394]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:21.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:21 smithi150 ceph-mon[22676]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:23.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:23 smithi130 ceph-mon[21867]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:23.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:23 smithi130 ceph-mon[26394]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:23.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:23 smithi150 ceph-mon[22676]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:25.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:25 smithi130 ceph-mon[26394]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:25.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:25 smithi130 ceph-mon[21867]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:25.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:25 smithi150 ceph-mon[22676]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:27.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:27 smithi130 ceph-mon[21867]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:27.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:27 smithi130 ceph-mon[21867]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:27.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:27 smithi130 ceph-mon[26394]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:27.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:27 smithi130 ceph-mon[26394]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:27.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:27 smithi150 ceph-mon[22676]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:27.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:27 smithi150 ceph-mon[22676]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:29.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:26:28] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-22T21:26:29.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:29 smithi130 ceph-mon[26394]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:29.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:29 smithi130 ceph-mon[21867]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:29.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:29 smithi150 ceph-mon[22676]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:31.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:31 smithi130 ceph-mon[26394]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:31.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:31 smithi130 ceph-mon[21867]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:31.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:31 smithi150 ceph-mon[22676]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:33.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:33 smithi130 ceph-mon[21867]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:33.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:33 smithi130 ceph-mon[26394]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:33.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:33 smithi150 ceph-mon[22676]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:34.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:34 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:26:34.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:34 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:26:34.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:34 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:26:35.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:35 smithi130 ceph-mon[26394]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:35.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:35 smithi130 ceph-mon[21867]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:35.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:35 smithi150 ceph-mon[22676]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:37.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:37 smithi130 ceph-mon[26394]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:37.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:37 smithi130 ceph-mon[26394]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:37.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:37 smithi130 ceph-mon[21867]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:37.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:37 smithi130 ceph-mon[21867]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:37.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:37 smithi150 ceph-mon[22676]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:37.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:37 smithi150 ceph-mon[22676]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:39.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:26:38] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-04-22T21:26:39.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:39 smithi130 ceph-mon[26394]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:39.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:39 smithi130 ceph-mon[21867]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:39.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:39 smithi150 ceph-mon[22676]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:41.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:41 smithi130 ceph-mon[26394]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:41.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:41 smithi130 ceph-mon[21867]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:41.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:41 smithi150 ceph-mon[22676]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:43.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:43 smithi130 ceph-mon[26394]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:43.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:43 smithi130 ceph-mon[21867]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:43.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:43 smithi150 ceph-mon[22676]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:45.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:45 smithi130 ceph-mon[21867]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:45.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:45 smithi130 ceph-mon[26394]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:45.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:45 smithi150 ceph-mon[22676]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:46.496 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.496 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.496 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.496 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.496 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:26:46.496 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:26:46.497 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.497 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:26:46.497 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:26:46.497 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.497 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.497 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.497 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.497 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:26:46.497 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:26:46.498 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.498 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:26:46.498 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:46 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:26:46.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:46 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:46 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:46 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:46 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:46 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:26:46.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:46 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:26:46.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:46 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:26:46.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:46 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:26:46.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:46 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:26:47.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:47 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:26:47.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:47 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:26:47.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:47 smithi130 ceph-mon[26394]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:47.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:47 smithi130 ceph-mon[26394]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:47.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:47 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:26:47.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:47 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:26:47.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:47 smithi130 ceph-mon[21867]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:47.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:47 smithi130 ceph-mon[21867]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:47.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:47 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:26:47.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:47 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:26:47.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:47 smithi150 ceph-mon[22676]: from='client.25213 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:26:47.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:47 smithi150 ceph-mon[22676]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:49.202 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:26:48] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-22T21:26:49.483 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:26:49.217+0000 7f6362244700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:26:49.484 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:26:49.484 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:26:49.484 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:26:49.484 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:26:49.484 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:49.484 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:49.484 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:26:49.484 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:26:49.484 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:26:49.484 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:26:49.484 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:26:49.484 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:26:49.485 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:26:49.485 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:26:49.485 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:26:49.485 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:26:49.485 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:26:49.485 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:26:49.485 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:49.486 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:49.486 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:26:49.486 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:26:49.486 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:26:49.486 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:26:49.486 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:26:49.486 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:26:49.486 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:26:49.486 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:26:49.486 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return future.result(timeout) 2024-04-22T21:26:49.486 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:26:49.486 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.__get_result() 2024-04-22T21:26:49.486 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:26:49.487 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: raise self._exception 2024-04-22T21:26:49.487 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:26:49.487 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: use_current_daemon_image=reconfig, 2024-04-22T21:26:49.487 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:26:49.487 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:26:49.487 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:26:49.487 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:26:49.487 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:26:49.487 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:26:49.487 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:26:49.487 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:49.487 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:49.487 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:26:49.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:26:49.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:26:49.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:26:49.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:26:49.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:26:49.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:26:49.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:26:49.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:26:49.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:26:49.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:26:49.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:26:49.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:49.488 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:49.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-mon[26394]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:49.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:26:49.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-mon[21867]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:49.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:49 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:26:49.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:49 smithi150 ceph-mon[22676]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:26:49.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:49 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:26:50.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:50 smithi150 ceph-mon[22676]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:26:50.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:26:50.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: return future.result(timeout) 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:26:50.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.__get_result() 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: raise self._exception 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:26:50.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:26:50.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:26:50.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:26:50.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:26:50.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:26:50.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:26:50.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:26:50.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:26:50.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:26:50.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:26:50.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:26:50.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:50.606 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:50.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:50 smithi130 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:26:50.798 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:26:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:26:50.800 INFO:journalctl@ceph.mon.a.smithi130.stdout: return future.result(timeout) 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.__get_result() 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: raise self._exception 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:26:50.801 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:50.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:50.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:26:50.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:26:50.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:26:50.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:26:50.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:26:50.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:26:50.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:26:50.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:26:50.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:26:50.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:26:50.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:26:50.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:26:50.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:50.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:50.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:50 smithi130 ceph-mon[26394]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:26:50.803 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:26:50.803 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:26:50.803 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:26:50.803 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:26:50.803 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:26:50.804 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:50.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:50.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:26:50.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:26:50.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:26:50.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:26:50.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:26:50.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:26:50.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:26:50.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:26:50.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: return future.result(timeout) 2024-04-22T21:26:50.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:26:50.805 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.__get_result() 2024-04-22T21:26:50.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:26:50.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: raise self._exception 2024-04-22T21:26:50.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:26:50.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:26:50.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:26:50.806 INFO:journalctl@ceph.mon.c.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:26:50.807 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:26:50.808 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:26:50.808 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:26:50.808 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:26:50.808 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:26:50.808 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:26:50.808 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:26:50.808 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:26:51.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:51 smithi130 ceph-mon[21867]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:51.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:51 smithi130 ceph-mon[26394]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:51.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:51 smithi150 ceph-mon[22676]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:26:53.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:53 smithi130 ceph-mon[21867]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:26:53.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:53 smithi130 ceph-mon[26394]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:26:53.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:53 smithi150 ceph-mon[22676]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:26:55.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:55 smithi130 ceph-mon[26394]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:26:55.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:55 smithi130 ceph-mon[21867]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:26:55.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:55 smithi150 ceph-mon[22676]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:26:57.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:57 smithi130 ceph-mon[26394]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:57.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:57 smithi130 ceph-mon[21867]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:57.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:57 smithi150 ceph-mon[22676]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:59.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:26:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:26:58] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-22T21:26:59.569 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:26:59 smithi130 ceph-mon[21867]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:59.569 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:26:59 smithi130 ceph-mon[26394]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:26:59.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:26:59 smithi150 ceph-mon[22676]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:27:01.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:01 smithi130 ceph-mon[26394]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:27:01.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:01 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1012396105' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:27:01.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:01 smithi130 ceph-mon[21867]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:27:01.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:01 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1012396105' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:27:01.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:01 smithi150 ceph-mon[22676]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:27:01.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:01 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1012396105' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:27:03.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:03 smithi130 ceph-mon[21867]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:27:03.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:03 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:27:03.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:03 smithi130 ceph-mon[26394]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:27:03.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:03 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:27:03.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:03 smithi150 ceph-mon[22676]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:27:03.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:03 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:27:05.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:05 smithi130 ceph-mon[26394]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:27:05.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:05 smithi130 ceph-mon[21867]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:27:05.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:05 smithi150 ceph-mon[22676]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:27:07.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:07 smithi130 ceph-mon[26394]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:27:07.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:07 smithi130 ceph-mon[21867]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:27:07.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:07 smithi150 ceph-mon[22676]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:27:09.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:27:08] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-04-22T21:27:09.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:09 smithi130 ceph-mon[26394]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:27:09.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:09 smithi130 ceph-mon[21867]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:27:09.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:09 smithi150 ceph-mon[22676]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:27:11.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:11 smithi130 ceph-mon[26394]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:27:11.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:11 smithi130 ceph-mon[26394]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:11.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:11 smithi130 ceph-mon[21867]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:27:11.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:11 smithi130 ceph-mon[21867]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:11.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:11 smithi150 ceph-mon[22676]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:27:11.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:11 smithi150 ceph-mon[22676]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:13.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:13 smithi130 ceph-mon[26394]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:27:13.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:13 smithi130 ceph-mon[21867]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:27:13.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:13 smithi150 ceph-mon[22676]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:27:15.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:15 smithi130 ceph-mon[26394]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:15.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:15 smithi130 ceph-mon[21867]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:15.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:15 smithi150 ceph-mon[22676]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:17.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:17 smithi130 ceph-mon[26394]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:17.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:17 smithi130 ceph-mon[21867]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:17.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:17 smithi150 ceph-mon[22676]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:18.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:18 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:27:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:18 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:27:18.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:18 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:27:19.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:27:18] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-22T21:27:19.753 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:19 smithi150 ceph-mon[22676]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:19.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:19 smithi130 ceph-mon[21867]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:19.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:19 smithi130 ceph-mon[26394]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:21.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:21 smithi130 ceph-mon[26394]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:21.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:21 smithi130 ceph-mon[26394]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:21.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:21 smithi130 ceph-mon[21867]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:21.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:21 smithi130 ceph-mon[21867]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:21.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:21 smithi150 ceph-mon[22676]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:21.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:21 smithi150 ceph-mon[22676]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:23.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:23 smithi130 ceph-mon[21867]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:23.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:23 smithi130 ceph-mon[26394]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:23.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:23 smithi150 ceph-mon[22676]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:25.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:25 smithi130 ceph-mon[26394]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:25.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:25 smithi130 ceph-mon[21867]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:25.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:25 smithi150 ceph-mon[22676]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:27.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:27 smithi130 ceph-mon[26394]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:27.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:27 smithi130 ceph-mon[21867]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:27.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:27 smithi150 ceph-mon[22676]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:29.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:27:28] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-22T21:27:29.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:29 smithi130 ceph-mon[26394]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:29.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:29 smithi130 ceph-mon[21867]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:29.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:29 smithi150 ceph-mon[22676]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:31.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:31 smithi130 ceph-mon[26394]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:31.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:31 smithi130 ceph-mon[26394]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:31.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:31 smithi130 ceph-mon[21867]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:31.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:31 smithi130 ceph-mon[21867]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:31.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:31 smithi150 ceph-mon[22676]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:31.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:31 smithi150 ceph-mon[22676]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:33.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:33 smithi130 ceph-mon[26394]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:33.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:33 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:27:33.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:33 smithi130 ceph-mon[21867]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:33.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:33 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:27:33.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:33 smithi150 ceph-mon[22676]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:33.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:33 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:27:35.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:35 smithi130 ceph-mon[21867]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:35.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:35 smithi130 ceph-mon[26394]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:35.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:35 smithi150 ceph-mon[22676]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:37.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:37 smithi130 ceph-mon[26394]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:37.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:37 smithi130 ceph-mon[21867]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:37.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:37 smithi150 ceph-mon[22676]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:39.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:27:38] "GET /metrics HTTP/1.1" 200 37918 "" "Prometheus/2.33.4" 2024-04-22T21:27:39.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:39 smithi130 ceph-mon[26394]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:39.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:39 smithi130 ceph-mon[21867]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:39.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:39 smithi150 ceph-mon[22676]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:41.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:41 smithi130 ceph-mon[21867]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:41.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:41 smithi130 ceph-mon[21867]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:41.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:41 smithi130 ceph-mon[26394]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:41.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:41 smithi130 ceph-mon[26394]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:41.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:41 smithi150 ceph-mon[22676]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:41.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:41 smithi150 ceph-mon[22676]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:43.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:43 smithi130 ceph-mon[26394]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:43.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:43 smithi130 ceph-mon[21867]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:43.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:43 smithi150 ceph-mon[22676]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:45.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:45 smithi130 ceph-mon[26394]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:45.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:45 smithi130 ceph-mon[21867]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:45.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:45 smithi150 ceph-mon[22676]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:47.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:47 smithi130 ceph-mon[26394]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:47.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:47 smithi130 ceph-mon[21867]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:47.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:47 smithi150 ceph-mon[22676]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:48.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:48 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:27:48.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:48 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:27:48.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:48 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:27:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:27:48] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.33.4" 2024-04-22T21:27:49.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:49 smithi130 ceph-mon[21867]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:49.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:49 smithi130 ceph-mon[26394]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:49.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:49 smithi150 ceph-mon[22676]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:50.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:50 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:27:50.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:50 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:27:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:50 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:27:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:50 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:27:50.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:50 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:27:50.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:50 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:27:50.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:50 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:27:50.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:50 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:27:50.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:50 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:27:50.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:50 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:27:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:50 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:27:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:50 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:27:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:50 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' 2024-04-22T21:27:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:50 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:27:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:50 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:27:51.728 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:51 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:27:51.728 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:51 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:27:51.728 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:51 smithi130 ceph-mon[21867]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:51.728 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:51 smithi130 ceph-mon[21867]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:51.728 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:51 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:27:51.729 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:51 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:27:51.729 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:51 smithi130 ceph-mon[26394]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:51.729 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:51 smithi130 ceph-mon[26394]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:51.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:51 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:27:51.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:51 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:27:51.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:51 smithi150 ceph-mon[22676]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:51.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:51 smithi150 ceph-mon[22676]: from='client.25240 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:27:53.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-mon[21867]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:53.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-mon[26394]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:53.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:27:53.435+0000 7f6362244700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:27:53.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:27:53.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:27:53.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:27:53.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:27:53.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:53.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:53.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:27:53.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:27:53.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:27:53.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return future.result(timeout) 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.__get_result() 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: raise self._exception 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: use_current_daemon_image=reconfig, 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:27:53.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:27:53.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:27:53.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:27:53.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:27:53.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:27:53.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:53.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:53.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:27:53.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:27:53.802 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:27:53.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:27:53.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:27:53.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:27:53.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:27:53.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:27:53.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:27:53.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:27:53.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:27:53.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:27:53.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:53.803 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:53.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:53 smithi150 ceph-mon[22676]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:27:54.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:54 smithi130 ceph-mon[26394]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:27:54.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:27:54.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:27:54.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:27:54.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:27:54.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:54.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:54.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:27:54.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: return future.result(timeout) 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.__get_result() 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: raise self._exception 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:27:54.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:27:54.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:27:54.802 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:27:54.802 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:27:54.802 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:27:54.802 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:27:54.802 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:27:54.802 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:54.802 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:54.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:54 smithi130 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:27:54.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:27:54.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:27:54.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:27:54.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:27:54.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:54.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:54.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:27:54.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:27:54.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:27:54.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:27:54.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:27:54.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:27:54.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: return future.result(timeout) 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.__get_result() 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: raise self._exception 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:27:54.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:27:54.807 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:27:54.808 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:27:54.808 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:54.808 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:54.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:54 smithi150 ceph-mon[22676]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:27:54.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:27:54.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:27:54.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:27:54.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:27:54.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:54.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:54.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:27:54.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:27:54.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:27:54.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:27:54.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:27:54.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: return future.result(timeout) 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.__get_result() 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: raise self._exception 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:27:54.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:27:54.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:27:54.856 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:27:54.856 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:27:54.856 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:27:54.856 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:27:54.856 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:27:55.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:55 smithi130 ceph-mon[26394]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:55.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:55 smithi130 ceph-mon[21867]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:55.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:55 smithi150 ceph-mon[22676]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:27:57.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:57 smithi130 ceph-mon[21867]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:27:57.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:57 smithi130 ceph-mon[26394]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:27:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:57 smithi150 ceph-mon[22676]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:27:59.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:27:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:27:58] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.33.4" 2024-04-22T21:27:59.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:27:59 smithi130 ceph-mon[21867]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:27:59.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:27:59 smithi130 ceph-mon[26394]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:27:59.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:27:59 smithi150 ceph-mon[22676]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:28:01.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:01 smithi130 ceph-mon[21867]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:28:01.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:01 smithi130 ceph-mon[26394]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:28:01.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:01 smithi150 ceph-mon[22676]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:28:03.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:03 smithi130 ceph-mon[26394]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:28:03.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:03 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:03.732 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:03 smithi130 ceph-mon[21867]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:28:03.732 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:03 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:03.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:03 smithi150 ceph-mon[22676]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:28:03.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:03 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:05.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:05 smithi130 ceph-mon[21867]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:28:05.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:05 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1072841547' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:28:05.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:05 smithi130 ceph-mon[26394]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:28:05.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:05 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1072841547' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:28:05.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:05 smithi150 ceph-mon[22676]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:28:05.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:05 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1072841547' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:28:07.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:07 smithi130 ceph-mon[21867]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:07.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:07 smithi130 ceph-mon[26394]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:07.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:07 smithi150 ceph-mon[22676]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:09.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:28:08] "GET /metrics HTTP/1.1" 200 37922 "" "Prometheus/2.33.4" 2024-04-22T21:28:09.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:09 smithi130 ceph-mon[26394]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:09.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:09 smithi130 ceph-mon[21867]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:09.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:09 smithi150 ceph-mon[22676]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:11.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:11 smithi130 ceph-mon[21867]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:28:11.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:11 smithi130 ceph-mon[26394]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:28:11.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:11 smithi150 ceph-mon[22676]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:28:13.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:13 smithi130 ceph-mon[21867]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:28:13.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:13 smithi130 ceph-mon[26394]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:28:13.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:13 smithi150 ceph-mon[22676]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:28:15.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:15 smithi130 ceph-mon[21867]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:15.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:15 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:15.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:15 smithi130 ceph-mon[26394]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:15.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:15 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:15.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:15 smithi150 ceph-mon[22676]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:15.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:15 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:17.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:17 smithi130 ceph-mon[21867]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:17.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:17 smithi130 ceph-mon[26394]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:17.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:17 smithi150 ceph-mon[22676]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:18 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:18.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:18 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:18.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:18 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:19.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:28:18] "GET /metrics HTTP/1.1" 200 37916 "" "Prometheus/2.33.4" 2024-04-22T21:28:19.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:19 smithi130 ceph-mon[21867]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:19.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:19 smithi130 ceph-mon[26394]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:19.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:19 smithi150 ceph-mon[22676]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:21.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:21 smithi130 ceph-mon[21867]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:28:21.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:21 smithi130 ceph-mon[26394]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:28:21.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:21 smithi150 ceph-mon[22676]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:28:22.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps' 2024-04-22T21:28:23.752 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:23 smithi130 ceph-mon[21867]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:23.753 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:23 smithi130 ceph-mon[26394]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:23.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:23 smithi150 ceph-mon[22676]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:24.251 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:28:24.251 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (9m) 2m ago 17m 24.7M - 0.23.0 ba2b418f427c 23fd04f877ce 2024-04-22T21:28:24.251 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (8m) 2m ago 16m 49.7M - 8.3.5 dad864ee21e9 e021384d6729 2024-04-22T21:28:24.251 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (7m) 2m ago 16m 58.5M - 3.5 e1d6a67b021e 0a30d8841db2 2024-04-22T21:28:24.251 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283 running (12m) 2m ago 21m 448M - 19.0.0-3198-g56e81df3 82086b19bb07 327605fe868f 2024-04-22T21:28:24.251 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (5m) 2m ago 23m 475M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:28:24.251 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (23m) 2m ago 23m 62.9M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (22m) 2m ago 22m 47.9M 2048M 17.2.0 e1d6a67b021e 861909a68cf9 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (22m) 2m ago 22m 46.9M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (9m) 2m ago 17m 15.1M - 1.3.1 1dbe0e931976 4057e105dc28 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (9m) 2m ago 17m 17.7M - 1.3.1 1dbe0e931976 2bcf91466257 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (21m) 2m ago 21m 53.4M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (21m) 2m ago 21m 55.8M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (20m) 2m ago 20m 49.7M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (20m) 2m ago 20m 52.1M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (19m) 2m ago 19m 52.6M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (19m) 2m ago 19m 51.7M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (19m) 2m ago 19m 53.2M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (18m) 2m ago 18m 52.7M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (2m) 2m ago 17m 44.2M - 2.33.4 514e6a882f6e 33d2ba18f840 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (16m) 2m ago 16m 75.1M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:28:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (16m) 2m ago 16m 73.0M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:28:24.879 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions' 2024-04-22T21:28:25.451 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:25 smithi130 ceph-mon[21867]: from='client.15387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:28:25.451 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:25 smithi130 ceph-mon[21867]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:28:25.451 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:25 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:25.451 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:25 smithi130 ceph-mon[26394]: from='client.15387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:28:25.451 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:25 smithi130 ceph-mon[26394]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:28:25.451 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:25 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:25.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:25 smithi150 ceph-mon[22676]: from='client.15387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:28:25.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:25 smithi150 ceph-mon[22676]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:28:25.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:25 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:26.979 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": {}, 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:28:26.980 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:28:26.981 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T21:28:26.981 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:28:26.981 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:28:26.981 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:28:27.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:27 smithi130 ceph-mon[21867]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:27.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:27 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3117501982' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:28:27.620 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:27 smithi130 ceph-mon[26394]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:27.620 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:27 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3117501982' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:28:27.622 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph -s' 2024-04-22T21:28:27.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:27 smithi150 ceph-mon[22676]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:27.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:27 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3117501982' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:28:29.122 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:28:28] "GET /metrics HTTP/1.1" 200 37916 "" "Prometheus/2.33.4" 2024-04-22T21:28:29.613 INFO:teuthology.orchestra.run.smithi130.stdout: cluster: 2024-04-22T21:28:29.613 INFO:teuthology.orchestra.run.smithi130.stdout: id: dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:28:29.613 INFO:teuthology.orchestra.run.smithi130.stdout: health: HEALTH_OK 2024-04-22T21:28:29.613 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:28:29.613 INFO:teuthology.orchestra.run.smithi130.stdout: services: 2024-04-22T21:28:29.613 INFO:teuthology.orchestra.run.smithi130.stdout: mon: 3 daemons, quorum a,c,b (age 22m) 2024-04-22T21:28:29.613 INFO:teuthology.orchestra.run.smithi130.stdout: mgr: y(active, since 3m), standbys: x 2024-04-22T21:28:29.613 INFO:teuthology.orchestra.run.smithi130.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-04-22T21:28:29.613 INFO:teuthology.orchestra.run.smithi130.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-22T21:28:29.614 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:28:29.614 INFO:teuthology.orchestra.run.smithi130.stdout: data: 2024-04-22T21:28:29.614 INFO:teuthology.orchestra.run.smithi130.stdout: pools: 6 pools, 161 pgs 2024-04-22T21:28:29.614 INFO:teuthology.orchestra.run.smithi130.stdout: objects: 209 objects, 585 KiB 2024-04-22T21:28:29.614 INFO:teuthology.orchestra.run.smithi130.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:29.614 INFO:teuthology.orchestra.run.smithi130.stdout: pgs: 161 active+clean 2024-04-22T21:28:29.615 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:28:29.615 INFO:teuthology.orchestra.run.smithi130.stdout: io: 2024-04-22T21:28:29.615 INFO:teuthology.orchestra.run.smithi130.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-22T21:28:29.615 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-04-22T21:28:29.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:29 smithi130 ceph-mon[26394]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:29.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:29 smithi130 ceph-mon[21867]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:29.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:29 smithi150 ceph-mon[22676]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:30.260 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph health detail' 2024-04-22T21:28:30.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:30 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1179753167' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:28:30.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:30 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1179753167' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:28:30.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:30 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1179753167' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T21:28:31.745 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:31 smithi130 ceph-mon[21867]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:28:31.745 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:31 smithi130 ceph-mon[26394]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:28:31.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:31 smithi150 ceph-mon[22676]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:28:32.283 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-04-22T21:28:32.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:32 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1433779865' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:28:32.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:32 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1433779865' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:28:32.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:32 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1433779865' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:28:33.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-04-22T21:28:33.635 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:33 smithi130 ceph-mon[26394]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:33.636 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:33 smithi130 ceph-mon[26394]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:33.636 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:33 smithi130 ceph-mon[21867]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:33.636 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:33 smithi130 ceph-mon[21867]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:33.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:33 smithi150 ceph-mon[22676]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:33.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:33 smithi150 ceph-mon[22676]: from='mgr.25114 172.21.15.130:0/3808487789' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:35.215 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:28:35.277 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:35 smithi130 ceph-mon[21867]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:28:35.277 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:35 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:35.277 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:35 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1956410055' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:28:35.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:35 smithi130 ceph-mon[26394]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:28:35.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:35 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:35.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:35 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1956410055' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:28:35.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:35 smithi150 ceph-mon[22676]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:28:35.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:35 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:35.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:35 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1956410055' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:28:35.880 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph mgr fail' 2024-04-22T21:28:37.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:37 smithi130 ceph-mon[21867]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:37.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:37 smithi130 ceph-mon[26394]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:37.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:37 smithi150 ceph-mon[22676]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:28:38.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ignoring --setuser ceph since I am not root 2024-04-22T21:28:38.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ignoring --setgroup ceph since I am not root 2024-04-22T21:28:38.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:38 smithi130 ceph-mgr[60104]: -- 172.21.15.130:0/2383306637 <== mon.1 v2:172.21.15.130:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b810e405a0 con 0x55b810e14800 2024-04-22T21:28:38.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:38.726+0000 7fde58a02200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:28:38.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:38 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3965678722' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T21:28:38.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:38 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T21:28:38.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:38 smithi130 ceph-mon[26394]: osdmap e105: 8 total, 8 up, 8 in 2024-04-22T21:28:38.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:38 smithi130 ceph-mon[26394]: Standby manager daemon y started 2024-04-22T21:28:38.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:38 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3965678722' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T21:28:38.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:38 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T21:28:38.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:38 smithi130 ceph-mon[21867]: osdmap e105: 8 total, 8 up, 8 in 2024-04-22T21:28:38.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:38 smithi130 ceph-mon[21867]: Standby manager daemon y started 2024-04-22T21:28:38.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:38 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3965678722' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T21:28:38.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:38 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T21:28:38.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:38 smithi150 ceph-mon[22676]: osdmap e105: 8 total, 8 up, 8 in 2024-04-22T21:28:38.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:38 smithi150 ceph-mon[22676]: Standby manager daemon y started 2024-04-22T21:28:38.852 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:28:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:28:38] ENGINE Bus STOPPING 2024-04-22T21:28:38.852 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:28:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:28:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T21:28:38.852 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:28:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:28:38] ENGINE Bus STOPPED 2024-04-22T21:28:38.852 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:28:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:28:38] ENGINE Bus STARTING 2024-04-22T21:28:38.852 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:28:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:28:38] ENGINE Serving on http://:::9283 2024-04-22T21:28:38.852 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:28:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:28:38] ENGINE Bus STARTED 2024-04-22T21:28:39.280 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:38.919+0000 7fde58a02200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:28:39.280 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:39.029+0000 7fde58a02200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:28:39.306 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'sleep 180' 2024-04-22T21:28:39.547 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:39.362+0000 7fde58a02200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:28:39.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:39.464+0000 7fde58a02200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:28:39.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T21:28:39.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: mgrmap e33: x(active, starting, since 0.635326s), standbys: y 2024-04-22T21:28:39.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:28:39.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:28:39.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:28:39.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:28:39.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: Manager daemon x is now available 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:39.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:28:39.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:28:39.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T21:28:39.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T21:28:39.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: [22/Apr/2024:21:28:39] ENGINE Bus STARTING 2024-04-22T21:28:39.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: [22/Apr/2024:21:28:39] ENGINE Serving on https://172.21.15.150:7150 2024-04-22T21:28:39.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: [22/Apr/2024:21:28:39] ENGINE Serving on http://172.21.15.150:8765 2024-04-22T21:28:39.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[26394]: [22/Apr/2024:21:28:39] ENGINE Bus STARTED 2024-04-22T21:28:39.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T21:28:39.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: mgrmap e33: x(active, starting, since 0.635326s), standbys: y 2024-04-22T21:28:39.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:28:39.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:28:39.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:28:39.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:28:39.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:28:39.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:28:39.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: Manager daemon x is now available 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:28:39.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T21:28:39.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T21:28:39.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: [22/Apr/2024:21:28:39] ENGINE Bus STARTING 2024-04-22T21:28:39.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: [22/Apr/2024:21:28:39] ENGINE Serving on https://172.21.15.150:7150 2024-04-22T21:28:39.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: [22/Apr/2024:21:28:39] ENGINE Serving on http://172.21.15.150:8765 2024-04-22T21:28:39.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:39 smithi130 ceph-mon[21867]: [22/Apr/2024:21:28:39] ENGINE Bus STARTED 2024-04-22T21:28:39.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T21:28:39.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: mgrmap e33: x(active, starting, since 0.635326s), standbys: y 2024-04-22T21:28:39.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:28:39.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:28:39.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:28:39.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:28:39.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:28:39.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:28:39.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: Manager daemon x is now available 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T21:28:39.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T21:28:39.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: [22/Apr/2024:21:28:39] ENGINE Bus STARTING 2024-04-22T21:28:39.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: [22/Apr/2024:21:28:39] ENGINE Serving on https://172.21.15.150:7150 2024-04-22T21:28:39.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: [22/Apr/2024:21:28:39] ENGINE Serving on http://172.21.15.150:8765 2024-04-22T21:28:39.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:39 smithi150 ceph-mon[22676]: [22/Apr/2024:21:28:39] ENGINE Bus STARTED 2024-04-22T21:28:40.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:40.707+0000 7fde58a02200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:28:40.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:40 smithi130 ceph-mon[26394]: mgrmap e34: x(active, since 1.64734s), standbys: y 2024-04-22T21:28:40.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:40 smithi130 ceph-mon[26394]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:40.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:40 smithi130 ceph-mon[21867]: mgrmap e34: x(active, since 1.64734s), standbys: y 2024-04-22T21:28:40.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:40 smithi130 ceph-mon[21867]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:40.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:40 smithi150 ceph-mon[22676]: mgrmap e34: x(active, since 1.64734s), standbys: y 2024-04-22T21:28:40.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:40 smithi150 ceph-mon[22676]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:41.293 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:40.898+0000 7fde58a02200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:28:41.293 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:41 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:41.074+0000 7fde58a02200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:28:41.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:41 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:41.379+0000 7fde58a02200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:28:41.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:41 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:41.485+0000 7fde58a02200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:28:41.670 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:41 smithi150 ceph-mon[22676]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:41.670 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:41 smithi150 ceph-mon[22676]: mgrmap e35: x(active, since 2s), standbys: y 2024-04-22T21:28:41.881 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:41 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:41.699+0000 7fde58a02200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:28:41.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:41 smithi130 ceph-mon[21867]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:41.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:41 smithi130 ceph-mon[21867]: mgrmap e35: x(active, since 2s), standbys: y 2024-04-22T21:28:41.882 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:41 smithi130 ceph-mon[26394]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:41.882 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:41 smithi130 ceph-mon[26394]: mgrmap e35: x(active, since 2s), standbys: y 2024-04-22T21:28:42.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:41 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:41.897+0000 7fde58a02200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:28:42.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:42 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:42.536+0000 7fde58a02200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:28:43.292 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:42.999+0000 7fde58a02200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:28:43.292 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:43.098+0000 7fde58a02200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:28:43.547 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:43.308+0000 7fde58a02200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T21:28:43.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:43 smithi150 ceph-mon[22676]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:43.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:43 smithi150 ceph-mon[22676]: mgrmap e36: x(active, since 4s), standbys: y 2024-04-22T21:28:43.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:43 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:43.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:43 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:43.852 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:43 smithi130 ceph-mon[21867]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:43.852 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:43 smithi130 ceph-mon[21867]: mgrmap e36: x(active, since 4s), standbys: y 2024-04-22T21:28:43.852 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:43 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:43.852 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:43 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:43.852 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:43 smithi130 ceph-mon[26394]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:43.852 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:43 smithi130 ceph-mon[26394]: mgrmap e36: x(active, since 4s), standbys: y 2024-04-22T21:28:43.852 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:43 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:43.852 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:43 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:44.226 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:43.867+0000 7fde58a02200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:28:44.226 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:43.963+0000 7fde58a02200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:28:44.547 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:44.241+0000 7fde58a02200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:28:44.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:44.380+0000 7fde58a02200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:28:44.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:44.483+0000 7fde58a02200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:28:44.850 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:44.610+0000 7fde58a02200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:28:45.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:44.929+0000 7fde58a02200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:28:45.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:45 smithi150 ceph-mon[22676]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:45.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:45 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:45.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:45 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:45.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:45 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:45.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:45 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:45.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:45 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:45.910 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-mon[26394]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:45.911 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:45.911 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:45.911 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:45.911 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:45.911 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:45.911 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-mon[21867]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:28:45.911 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:45.911 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:45.911 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:45.911 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:45.912 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:45.912 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:45.579+0000 7fde58a02200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:28:45.912 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:45.816+0000 7fde58a02200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:28:45.912 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:28:45] ENGINE Bus STARTING 2024-04-22T21:28:45.912 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CherryPy Checker: 2024-04-22T21:28:45.912 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: The Application mounted at '' has an empty config. 2024-04-22T21:28:45.912 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:28:45.912 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:28:45] ENGINE Serving on http://:::9283 2024-04-22T21:28:45.912 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:45 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:28:45] ENGINE Bus STARTED 2024-04-22T21:28:46.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: Standby manager daemon y restarted 2024-04-22T21:28:46.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: Standby manager daemon y started 2024-04-22T21:28:46.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T21:28:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:28:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T21:28:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:28:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:28:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:28:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:28:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:28:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:28:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:28:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:28:46.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:46 smithi150 ceph-mon[22676]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:28:47.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: Standby manager daemon y restarted 2024-04-22T21:28:47.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: Standby manager daemon y started 2024-04-22T21:28:47.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T21:28:47.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:28:47.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T21:28:47.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:28:47.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:47.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:47.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:28:47.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:28:47.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:28:47.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:28:47.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:28:47.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:28:47.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:28:47.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[21867]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:28:47.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: Standby manager daemon y restarted 2024-04-22T21:28:47.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: Standby manager daemon y started 2024-04-22T21:28:47.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T21:28:47.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:28:47.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T21:28:47.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:28:47.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:47.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:47.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:28:47.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:28:47.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:28:47.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:28:47.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:28:47.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:28:47.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:28:47.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:46 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:28:47.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:47 smithi150 ceph-mon[22676]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:28:47.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:47 smithi150 ceph-mon[22676]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:28:47.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:47 smithi150 ceph-mon[22676]: mgrmap e37: x(active, since 8s), standbys: y 2024-04-22T21:28:47.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:47 smithi150 ceph-mon[22676]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:28:47.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:47 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:47.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:47 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:47.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:47 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:47.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:47 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:47.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:47 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:47.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:47 smithi150 ceph-mon[22676]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:28:47.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:47 smithi150 ceph-mon[22676]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:28:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[21867]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:28:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[21867]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:28:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[21867]: mgrmap e37: x(active, since 8s), standbys: y 2024-04-22T21:28:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[21867]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:28:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[21867]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:28:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[21867]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:28:48.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[26394]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:28:48.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[26394]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:28:48.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[26394]: mgrmap e37: x(active, since 8s), standbys: y 2024-04-22T21:28:48.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[26394]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:28:48.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:48.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:48.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:48.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:48.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:48.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[26394]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:28:48.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:47 smithi130 ceph-mon[26394]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:28:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:28:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:28:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:28:50.439 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 systemd[1]: Stopping Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:28:50.439 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 bash[54450]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T21:28:50.439 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:50 smithi150 ceph-mon[22676]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:28:50.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:50 smithi130 ceph-mon[21867]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:28:50.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:50 smithi130 ceph-mon[26394]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:28:50.785 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:28:50.518Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T21:28:50.786 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:28:50.518Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T21:28:50.786 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:28:50.518Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T21:28:50.786 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:28:50.518Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T21:28:50.786 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:28:50.518Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T21:28:50.786 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:28:50.518Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T21:28:50.786 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:28:50.518Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T21:28:50.786 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:28:50.518Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T21:28:50.786 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:28:50.518Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T21:28:50.786 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:28:50.519Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T21:28:50.786 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:28:50.519Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T21:28:50.786 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[51527]: ts=2024-04-22T21:28:50.519Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T21:28:50.786 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 podman[54463]: 2024-04-22 21:28:50.528579797 +0000 UTC m=+0.126744932 container died 33d2ba18f8405b3b673795f007ade0c5a09cc6f5036e6eed1a1fd5b3de882359 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:28:51.101 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 podman[54463]: 2024-04-22 21:28:50.900725056 +0000 UTC m=+0.498890190 container cleanup 33d2ba18f8405b3b673795f007ade0c5a09cc6f5036e6eed1a1fd5b3de882359 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:28:51.101 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 bash[54463]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T21:28:51.101 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:50 smithi150 bash[54499]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T21:28:51.511 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 podman[54477]: 2024-04-22 21:28:51.176837136 +0000 UTC m=+0.650953310 container remove 33d2ba18f8405b3b673795f007ade0c5a09cc6f5036e6eed1a1fd5b3de882359 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:28:51.511 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T21:28:51.511 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 systemd[1]: Stopped Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:28:51.511 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.374s CPU time. 2024-04-22T21:28:51.511 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 systemd[1]: Starting Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:28:51.511 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 podman[54570]: 2024-04-22 21:28:51.314231059 +0000 UTC m=+0.023207526 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T21:28:51.511 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 podman[54570]: 2024-04-22 21:28:51.421880681 +0000 UTC m=+0.130857135 container create b7cd02579a7ff838594445bec5be9b78d92d5a9760c8062042e05d2aa234f294 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:28:51.777 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 podman[54570]: 2024-04-22 21:28:51.624111391 +0000 UTC m=+0.333087847 container init b7cd02579a7ff838594445bec5be9b78d92d5a9760c8062042e05d2aa234f294 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:28:51.777 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 podman[54570]: 2024-04-22 21:28:51.626821975 +0000 UTC m=+0.335798430 container start b7cd02579a7ff838594445bec5be9b78d92d5a9760c8062042e05d2aa234f294 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:28:51.777 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:51.651Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T21:28:51.777 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:51.651Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T21:28:51.777 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:51.651Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T21:28:51.777 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:51.651Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi150 (none))" 2024-04-22T21:28:51.777 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:51.651Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T21:28:51.778 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:51.651Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T21:28:51.778 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:51.653Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T21:28:51.778 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:51.653Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T21:28:51.778 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:51.654Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T21:28:51.778 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:51.656Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T21:28:51.778 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:51.656Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.785µs 2024-04-22T21:28:51.778 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:51.656Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T21:28:51.778 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 bash[54570]: b7cd02579a7ff838594445bec5be9b78d92d5a9760c8062042e05d2aa234f294 2024-04-22T21:28:51.778 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 systemd[1]: Started Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:28:52.050 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:51.859Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-04-22T21:28:52.050 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:28:51] ENGINE Bus STOPPING 2024-04-22T21:28:52.051 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:28:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T21:28:52.051 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:28:51] ENGINE Bus STOPPED 2024-04-22T21:28:52.051 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:28:51] ENGINE Bus STARTING 2024-04-22T21:28:52.051 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:28:51] ENGINE Serving on http://:::9283 2024-04-22T21:28:52.051 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:28:51 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:28:51] ENGINE Bus STARTED 2024-04-22T21:28:52.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-mon[22676]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:28:52.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:52.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:52.352 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:28:52.352 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2024-04-22T21:28:52.352 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:52.352 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:52.066Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-04-22T21:28:52.352 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:52.278Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-04-22T21:28:52.352 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:52.282Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-04-22T21:28:52.352 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:52.282Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-04-22T21:28:52.352 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:52.282Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=36.857µs wal_replay_duration=625.539831ms total_replay_duration=625.590045ms 2024-04-22T21:28:52.352 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:52.286Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T21:28:52.352 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:52.286Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T21:28:52.352 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:52.286Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T21:28:52.352 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:52.310Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.731229ms db_storage=1.326µs remote_storage=1.639µs web_handler=372ns query_engine=816ns scrape=503.661µs scrape_sd=190.693µs notify=18.579µs notify_sd=22.116µs rules=22.441078ms 2024-04-22T21:28:52.353 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:28:52 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:28:52.310Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T21:28:52.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:52 smithi130 ceph-mon[21867]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:28:52.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:52 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:52.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:52 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:52.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:52 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:28:52.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:52 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2024-04-22T21:28:52.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:52 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:52.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:52 smithi130 ceph-mon[26394]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:28:52.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:52 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:52.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:52 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:52.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:52 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:28:52.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:52 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2024-04-22T21:28:52.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:52 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:53.492 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:53 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:28:53.493 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:53 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2024-04-22T21:28:53.493 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:53 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:28:53.493 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:53 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2024-04-22T21:28:53.521 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:53 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:28:53.521 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:53 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2024-04-22T21:28:54.441 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:54 smithi150 ceph-mon[22676]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:28:54.441 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:54 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:54.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:54 smithi130 ceph-mon[26394]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:28:54.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:54 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:54.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:54 smithi130 ceph-mon[21867]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:28:54.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:54 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:28:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[21867]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:28:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:28:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:28:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:56.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[26394]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:28:56.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:56.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:56.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:56.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:28:56.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:28:56.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:56 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:56 smithi150 ceph-mon[22676]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:28:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:56 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:28:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:56 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:56 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:56 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:28:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:56 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:28:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:56 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:28:58.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:28:58 smithi130 ceph-mon[21867]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:28:58.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:28:58 smithi130 ceph-mon[26394]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:28:58.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:28:58 smithi150 ceph-mon[22676]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:28:59.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:28:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:28:59] "GET /metrics HTTP/1.1" 200 37918 "" "Prometheus/2.33.4" 2024-04-22T21:29:00.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:00 smithi130 ceph-mon[21867]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:00.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:00 smithi130 ceph-mon[26394]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:00.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:00 smithi150 ceph-mon[22676]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:02.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:02 smithi130 ceph-mon[21867]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:02.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:02 smithi130 ceph-mon[26394]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:02.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:02 smithi150 ceph-mon[22676]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:04.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:04 smithi130 ceph-mon[21867]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:04.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:04 smithi130 ceph-mon[26394]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:04.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:04 smithi150 ceph-mon[22676]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:06.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:06 smithi130 ceph-mon[26394]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:06.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:06 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:06.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:06 smithi130 ceph-mon[21867]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:06.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:06 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:06.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:06 smithi150 ceph-mon[22676]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:06.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:06 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:08.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:08 smithi130 ceph-mon[21867]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:08.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:08 smithi130 ceph-mon[26394]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:08.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:08 smithi150 ceph-mon[22676]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:09.516 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:09 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:29:09.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:09 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:29:09.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:09 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:29:09.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:29:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:29:09] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-04-22T21:29:10.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:10 smithi130 ceph-mon[26394]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:10.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:10 smithi130 ceph-mon[21867]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:10.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:10 smithi150 ceph-mon[22676]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:12.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:12 smithi130 ceph-mon[26394]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:12.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:12 smithi130 ceph-mon[21867]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:12.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:12 smithi150 ceph-mon[22676]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:14.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:14 smithi130 ceph-mon[26394]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:14.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:14 smithi130 ceph-mon[21867]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:14.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:14 smithi150 ceph-mon[22676]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:16.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:16 smithi130 ceph-mon[26394]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:16.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:16 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:16.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:16 smithi130 ceph-mon[21867]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:16.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:16 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:16.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:16 smithi150 ceph-mon[22676]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:16.772 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:16 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:18.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:18 smithi130 ceph-mon[26394]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:18.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:18 smithi130 ceph-mon[21867]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:18 smithi150 ceph-mon[22676]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:19.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:29:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:29:19] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-04-22T21:29:20.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:20 smithi130 ceph-mon[21867]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:20.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:20 smithi130 ceph-mon[26394]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:20.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:20 smithi150 ceph-mon[22676]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:22.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:22 smithi130 ceph-mon[26394]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:22.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:22 smithi130 ceph-mon[21867]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:22.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:22 smithi150 ceph-mon[22676]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:24.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:24 smithi130 ceph-mon[21867]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:24.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:24 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:29:24.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:24 smithi130 ceph-mon[26394]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:24.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:24 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:29:24.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:24 smithi150 ceph-mon[22676]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:24.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:24 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:29:26.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:26 smithi130 ceph-mon[26394]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:26.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:26 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:26.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:26 smithi130 ceph-mon[21867]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:26.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:26 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:26.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:26 smithi150 ceph-mon[22676]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:26.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:26 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:28.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:28 smithi130 ceph-mon[26394]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:28.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:28 smithi130 ceph-mon[21867]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:28.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:28 smithi150 ceph-mon[22676]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:29.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:29:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:29:29] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.33.4" 2024-04-22T21:29:30.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:30 smithi130 ceph-mon[21867]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:30.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:30 smithi130 ceph-mon[26394]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:30.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:30 smithi150 ceph-mon[22676]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:32.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:32 smithi130 ceph-mon[26394]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:32.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:32 smithi130 ceph-mon[21867]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:32.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:32 smithi150 ceph-mon[22676]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:34.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:34 smithi150 ceph-mon[22676]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:34.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:34 smithi130 ceph-mon[21867]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:34.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:34 smithi130 ceph-mon[26394]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:36.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:36 smithi150 ceph-mon[22676]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:36.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:36 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:36.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:36 smithi130 ceph-mon[21867]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:36.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:36 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:36.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:36 smithi130 ceph-mon[26394]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:36.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:36 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:38.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:38 smithi150 ceph-mon[22676]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:38.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:38 smithi130 ceph-mon[21867]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:38.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:38 smithi130 ceph-mon[26394]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:39.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:29:39.601 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:29:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:29:39] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-04-22T21:29:39.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:29:39.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:29:40.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:40 smithi150 ceph-mon[22676]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:40.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:40 smithi130 ceph-mon[21867]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:40.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:40 smithi130 ceph-mon[26394]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:42.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:42 smithi150 ceph-mon[22676]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:42.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:42 smithi130 ceph-mon[21867]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:42.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:42 smithi130 ceph-mon[26394]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:44.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:44 smithi150 ceph-mon[22676]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:44.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:44 smithi130 ceph-mon[21867]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:44.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:44 smithi130 ceph-mon[26394]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:46.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:46 smithi150 ceph-mon[22676]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:46.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:46 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:46.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:46 smithi130 ceph-mon[21867]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:46.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:46 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:46.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:46 smithi130 ceph-mon[26394]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:46.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:46 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:48.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:48 smithi150 ceph-mon[22676]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:48.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:48 smithi130 ceph-mon[21867]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:48.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:48 smithi130 ceph-mon[26394]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:49.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:29:49 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:29:49] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-04-22T21:29:50.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:50 smithi150 ceph-mon[22676]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:50.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:50 smithi130 ceph-mon[21867]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:50.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:50 smithi130 ceph-mon[26394]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:52.599 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:52 smithi130 ceph-mon[21867]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:52.600 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:52 smithi130 ceph-mon[26394]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:52.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:52 smithi150 ceph-mon[22676]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:54.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:54 smithi150 ceph-mon[22676]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:54.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:54 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:29:54.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:54 smithi130 ceph-mon[21867]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:54.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:54 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:29:54.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:54 smithi130 ceph-mon[26394]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:54.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:54 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:29:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:56 smithi150 ceph-mon[22676]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:56 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:56.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:56 smithi130 ceph-mon[26394]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:56.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:56 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:56.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:56 smithi130 ceph-mon[21867]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:29:56.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:56 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:29:57.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:57 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:29:57.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:57 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:29:57.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:57 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:29:57.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:57 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:29:57.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:57 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:29:57.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:57 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:29:57.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:57 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:29:57.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:57 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:29:57.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:57 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:29:58.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:29:58 smithi130 ceph-mon[26394]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:58.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:29:58 smithi130 ceph-mon[21867]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:58.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:29:58 smithi150 ceph-mon[22676]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:29:59.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:29:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:29:59] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.33.4" 2024-04-22T21:30:00.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:00 smithi130 ceph-mon[26394]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:00.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:00 smithi130 ceph-mon[26394]: overall HEALTH_OK 2024-04-22T21:30:00.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:00 smithi130 ceph-mon[21867]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:00.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:00 smithi130 ceph-mon[21867]: overall HEALTH_OK 2024-04-22T21:30:00.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:00 smithi150 ceph-mon[22676]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:00.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:00 smithi150 ceph-mon[22676]: overall HEALTH_OK 2024-04-22T21:30:02.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:02 smithi130 ceph-mon[26394]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:02.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:02 smithi130 ceph-mon[21867]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:02.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:02 smithi150 ceph-mon[22676]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:04.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:04 smithi130 ceph-mon[21867]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:04.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:04 smithi130 ceph-mon[26394]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:04.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:04 smithi150 ceph-mon[22676]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:06.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:06 smithi130 ceph-mon[21867]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:06.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:06 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:06.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:06 smithi130 ceph-mon[26394]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:06.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:06 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:06 smithi150 ceph-mon[22676]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:06 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:08.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:08 smithi130 ceph-mon[21867]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:08 smithi130 ceph-mon[26394]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:08.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:08 smithi150 ceph-mon[22676]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:09.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:09 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:30:09.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:09 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:30:09.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:09 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:30:09.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:30:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:30:09] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.33.4" 2024-04-22T21:30:10.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:10 smithi130 ceph-mon[26394]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:10.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:10 smithi130 ceph-mon[21867]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:10.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:10 smithi150 ceph-mon[22676]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:12.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:12 smithi130 ceph-mon[26394]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:12 smithi130 ceph-mon[21867]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:12.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:12 smithi150 ceph-mon[22676]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:14.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:14 smithi130 ceph-mon[21867]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:14.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:14 smithi130 ceph-mon[26394]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:14.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:14 smithi150 ceph-mon[22676]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:16.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:16 smithi130 ceph-mon[21867]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:16.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:16 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:16.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:16 smithi130 ceph-mon[26394]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:16.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:16 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:16.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:16 smithi150 ceph-mon[22676]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:16.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:16 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:18 smithi130 ceph-mon[26394]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:18.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:18 smithi130 ceph-mon[21867]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:18.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:18 smithi150 ceph-mon[22676]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:19.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:30:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:30:19] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.33.4" 2024-04-22T21:30:20.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:20 smithi130 ceph-mon[26394]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:20.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:20 smithi130 ceph-mon[21867]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:20.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:20 smithi150 ceph-mon[22676]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:22.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:22 smithi130 ceph-mon[21867]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:22.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:22 smithi130 ceph-mon[26394]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:22.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:22 smithi150 ceph-mon[22676]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:24.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:24 smithi130 ceph-mon[26394]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:24.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:24 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:30:24.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:24 smithi130 ceph-mon[21867]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:24.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:24 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:30:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:24 smithi150 ceph-mon[22676]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:24 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:30:26.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:26 smithi130 ceph-mon[21867]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:26.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:26 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:26.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:26 smithi130 ceph-mon[26394]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:26.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:26 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:26.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:26 smithi150 ceph-mon[22676]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:26.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:26 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:28.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:28 smithi130 ceph-mon[21867]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:28.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:28 smithi130 ceph-mon[26394]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:28.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:28 smithi150 ceph-mon[22676]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:29.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:30:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:30:29] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.33.4" 2024-04-22T21:30:30.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:30 smithi130 ceph-mon[21867]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:30.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:30 smithi130 ceph-mon[26394]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:30.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:30 smithi150 ceph-mon[22676]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:32.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:32 smithi130 ceph-mon[21867]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:32.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:32 smithi130 ceph-mon[26394]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:32.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:32 smithi150 ceph-mon[22676]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:34.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:34 smithi130 ceph-mon[26394]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:34.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:34 smithi130 ceph-mon[21867]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:34.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:34 smithi150 ceph-mon[22676]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:36.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:36 smithi130 ceph-mon[26394]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:36.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:36 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:36.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:36 smithi130 ceph-mon[21867]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:36.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:36 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:36.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:36 smithi150 ceph-mon[22676]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:36.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:36 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:38.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:38 smithi130 ceph-mon[21867]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:38.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:38 smithi130 ceph-mon[26394]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:38.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:38 smithi150 ceph-mon[22676]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:39.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:30:39.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:30:39.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:30:39.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:30:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:30:39] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.33.4" 2024-04-22T21:30:40.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:40 smithi130 ceph-mon[26394]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:40.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:40 smithi130 ceph-mon[21867]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:40.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:40 smithi150 ceph-mon[22676]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:42.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:42 smithi130 ceph-mon[26394]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:42.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:42 smithi130 ceph-mon[21867]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:42.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:42 smithi150 ceph-mon[22676]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:44.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:44 smithi130 ceph-mon[21867]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:44.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:44 smithi130 ceph-mon[26394]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:44.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:44 smithi150 ceph-mon[22676]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:46.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:46 smithi130 ceph-mon[26394]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:46.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:46 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:46.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:46 smithi130 ceph-mon[21867]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:46.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:46 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:46.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:46 smithi150 ceph-mon[22676]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:46.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:46 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:48.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:48 smithi130 ceph-mon[26394]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:48.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:48 smithi130 ceph-mon[21867]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:48.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:48 smithi150 ceph-mon[22676]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:49.771 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:30:49 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:30:49] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.33.4" 2024-04-22T21:30:50.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:50 smithi130 ceph-mon[26394]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:50.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:50 smithi130 ceph-mon[21867]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:50 smithi150 ceph-mon[22676]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:52.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:52 smithi130 ceph-mon[26394]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:52.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:52 smithi130 ceph-mon[21867]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:52.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:52 smithi150 ceph-mon[22676]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:54.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:54 smithi130 ceph-mon[26394]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:54.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:54 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:30:54.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:54 smithi130 ceph-mon[21867]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:54.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:54 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:30:54.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:54 smithi150 ceph-mon[22676]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:54.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:54 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:30:56.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:56 smithi130 ceph-mon[26394]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:56.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:56 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:56.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:56 smithi130 ceph-mon[21867]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:56.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:56 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:56.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:56 smithi150 ceph-mon[22676]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:30:56.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:56 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:30:58.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:58 smithi130 ceph-mon[21867]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:58.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:58 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:30:58.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:58 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:30:58.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:30:58 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:30:58.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:58 smithi130 ceph-mon[26394]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:58.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:58 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:30:58.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:58 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:30:58.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:30:58 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:30:58.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:58 smithi150 ceph-mon[22676]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:30:58.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:58 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:30:58.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:58 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:30:58.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:30:58 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:30:59.850 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:30:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:30:59] "GET /metrics HTTP/1.1" 200 37922 "" "Prometheus/2.33.4" 2024-04-22T21:31:00.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:00 smithi130 ceph-mon[21867]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:00.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:00 smithi130 ceph-mon[26394]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:00.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:00 smithi150 ceph-mon[22676]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:02.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:02 smithi130 ceph-mon[21867]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:02.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:02 smithi130 ceph-mon[26394]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:02.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:02 smithi150 ceph-mon[22676]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:04.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:04 smithi130 ceph-mon[26394]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:04.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:04 smithi130 ceph-mon[21867]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:04.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:04 smithi150 ceph-mon[22676]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:06.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:06 smithi130 ceph-mon[21867]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:06.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:06 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:06.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:06 smithi130 ceph-mon[26394]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:06.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:06 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:06 smithi150 ceph-mon[22676]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:06 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:08.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:08 smithi130 ceph-mon[21867]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:08.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:08 smithi130 ceph-mon[26394]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:08.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:08 smithi150 ceph-mon[22676]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:09.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:09 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:09.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:09 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:09.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:09 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:09.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:31:09] "GET /metrics HTTP/1.1" 200 37925 "" "Prometheus/2.33.4" 2024-04-22T21:31:10.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:10 smithi130 ceph-mon[21867]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:10.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:10 smithi130 ceph-mon[26394]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:10.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:10 smithi150 ceph-mon[22676]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:12 smithi130 ceph-mon[21867]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:12.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:12 smithi130 ceph-mon[26394]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:12.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:12 smithi150 ceph-mon[22676]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:14.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:14 smithi130 ceph-mon[21867]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:14.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:14 smithi130 ceph-mon[26394]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:14.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:14 smithi150 ceph-mon[22676]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:16.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:16 smithi130 ceph-mon[21867]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:16.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:16 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:16.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:16 smithi130 ceph-mon[26394]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:16.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:16 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:16.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:16 smithi150 ceph-mon[22676]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:16.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:16 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:18 smithi130 ceph-mon[26394]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:18.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:18 smithi130 ceph-mon[21867]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:18.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:18 smithi150 ceph-mon[22676]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:19.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:31:19] "GET /metrics HTTP/1.1" 200 37925 "" "Prometheus/2.33.4" 2024-04-22T21:31:20.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:20 smithi130 ceph-mon[26394]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:20.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:20 smithi130 ceph-mon[21867]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:20.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:20 smithi150 ceph-mon[22676]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:22.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:22 smithi130 ceph-mon[26394]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:22.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:22 smithi130 ceph-mon[21867]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:22.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:22 smithi150 ceph-mon[22676]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:24.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:24 smithi130 ceph-mon[26394]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:24.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:24 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:24.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:24 smithi130 ceph-mon[21867]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:24.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:24 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:24 smithi150 ceph-mon[22676]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:24 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:25.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:25 smithi150 ceph-mon[22676]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:25 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:26.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:25 smithi130 ceph-mon[26394]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:26.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:25 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:25 smithi130 ceph-mon[21867]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:25 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:28.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:27 smithi130 ceph-mon[26394]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:28.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:27 smithi130 ceph-mon[21867]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:28.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:27 smithi150 ceph-mon[22676]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:29.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:31:29] "GET /metrics HTTP/1.1" 200 37922 "" "Prometheus/2.33.4" 2024-04-22T21:31:30.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:30 smithi130 ceph-mon[21867]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:30.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:30 smithi130 ceph-mon[26394]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:30.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:30 smithi150 ceph-mon[22676]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:32.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:32 smithi130 ceph-mon[21867]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:32.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:32 smithi130 ceph-mon[26394]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:32.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:32 smithi150 ceph-mon[22676]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:34.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:34 smithi130 ceph-mon[21867]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:34.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:34 smithi130 ceph-mon[26394]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:34.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:34 smithi150 ceph-mon[22676]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:36.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:36 smithi130 ceph-mon[26394]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:36.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:36 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:36.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:36 smithi130 ceph-mon[21867]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:36.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:36 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:36.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:36 smithi150 ceph-mon[22676]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:36.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:36 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:38.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:38 smithi130 ceph-mon[26394]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:38.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:38 smithi130 ceph-mon[21867]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:38.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:38 smithi150 ceph-mon[22676]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:39.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:39 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:39.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:39 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:39.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:39 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:39.601 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:31:39] "GET /metrics HTTP/1.1" 200 37924 "" "Prometheus/2.33.4" 2024-04-22T21:31:40.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:40 smithi130 ceph-mon[21867]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:40.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:40 smithi130 ceph-mon[26394]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:40.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:40 smithi150 ceph-mon[22676]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:42.263 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps' 2024-04-22T21:31:42.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:42 smithi130 ceph-mon[21867]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:42.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:42 smithi130 ceph-mon[26394]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:42.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:42 smithi150 ceph-mon[22676]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:44.221 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:31:44.221 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (12m) 2m ago 20m 24.2M - 0.23.0 ba2b418f427c 23fd04f877ce 2024-04-22T21:31:44.221 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (12m) 2m ago 20m 50.2M - 8.3.5 dad864ee21e9 e021384d6729 2024-04-22T21:31:44.221 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (3m) 2m ago 19m 48.3M - 3.5 e1d6a67b021e 8cf00b4e4942 2024-04-22T21:31:44.222 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283 running (15m) 2m ago 25m 493M - 19.0.0-3198-g56e81df3 82086b19bb07 327605fe868f 2024-04-22T21:31:44.222 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (9m) 2m ago 26m 75.0M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:31:44.222 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (26m) 2m ago 26m 72.2M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:31:44.222 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (25m) 2m ago 25m 51.5M 2048M 17.2.0 e1d6a67b021e 861909a68cf9 2024-04-22T21:31:44.222 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (25m) 2m ago 25m 51.4M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:31:44.222 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (12m) 2m ago 21m 15.6M - 1.3.1 1dbe0e931976 4057e105dc28 2024-04-22T21:31:44.222 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (12m) 2m ago 21m 17.2M - 1.3.1 1dbe0e931976 2bcf91466257 2024-04-22T21:31:44.222 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (24m) 2m ago 24m 53.7M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:31:44.222 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (24m) 2m ago 24m 56.0M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:31:44.222 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (23m) 2m ago 23m 49.8M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:31:44.222 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (23m) 2m ago 23m 51.4M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:31:44.222 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (23m) 2m ago 23m 52.9M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:31:44.223 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (22m) 2m ago 22m 51.7M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:31:44.223 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (22m) 2m ago 22m 53.6M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:31:44.223 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (22m) 2m ago 21m 53.0M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:31:44.223 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (2m) 2m ago 21m 44.9M - 2.33.4 514e6a882f6e b7cd02579a7f 2024-04-22T21:31:44.223 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (19m) 2m ago 19m 76.3M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:31:44.223 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (19m) 2m ago 19m 74.0M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:31:44.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:44 smithi130 ceph-mon[21867]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:44.302 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:44 smithi130 ceph-mon[26394]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:44.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:44 smithi150 ceph-mon[22676]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:44.827 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ls' 2024-04-22T21:31:45.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:45 smithi130 ceph-mon[26394]: from='client.25336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:45.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:45 smithi130 ceph-mon[21867]: from='client.25336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:45.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:45 smithi150 ceph-mon[22676]: from='client.25336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:46.369 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:46 smithi130 ceph-mon[26394]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:46.369 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:46 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:46.370 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:46 smithi130 ceph-mon[21867]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:46.370 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:46 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:46.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:46 smithi150 ceph-mon[22676]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:46.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:46 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:46.775 INFO:teuthology.orchestra.run.smithi130.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-22T21:31:46.776 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager ?:9093,9094 1/1 3m ago 21m smithi130=a;count:1 2024-04-22T21:31:46.776 INFO:teuthology.orchestra.run.smithi130.stdout:grafana ?:3000 1/1 2m ago 21m smithi150=a;count:1 2024-04-22T21:31:46.776 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo ?:5000 1/1 3m ago 19m count:1 2024-04-22T21:31:46.776 INFO:teuthology.orchestra.run.smithi130.stdout:mgr 2/2 3m ago 25m smithi130=y;smithi150=x;count:2 2024-04-22T21:31:46.776 INFO:teuthology.orchestra.run.smithi130.stdout:mon 3/3 3m ago 25m smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi150:172.21.15.150=b;count:3 2024-04-22T21:31:46.776 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter ?:9100 2/2 3m ago 21m smithi130=a;smithi150=b;count:2 2024-04-22T21:31:46.776 INFO:teuthology.orchestra.run.smithi130.stdout:osd 8 3m ago - 2024-04-22T21:31:46.776 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus ?:9095 1/1 2m ago 21m smithi150=a;count:1 2024-04-22T21:31:46.776 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo ?:8000 2/2 3m ago 19m count:2 2024-04-22T21:31:47.492 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions' 2024-04-22T21:31:48.465 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:48 smithi130 ceph-mon[26394]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:48.465 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:48 smithi130 ceph-mon[26394]: from='client.25342 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:48.465 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:48 smithi130 ceph-mon[21867]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:48.466 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:48 smithi130 ceph-mon[21867]: from='client.25342 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:48.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:48 smithi150 ceph-mon[22676]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:48.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:48 smithi150 ceph-mon[22676]: from='client.25342 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:49.646 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:31:49.646 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:31:49.646 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T21:31:49.646 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:31:49.646 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:31:49.646 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:31:49.646 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:31:49.647 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:31:49.647 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:31:49.647 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:31:49.647 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": {}, 2024-04-22T21:31:49.647 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:31:49.647 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:31:49.647 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:31:49.647 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:31:49.647 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T21:31:49.647 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:31:49.647 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:31:49.647 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:31:49.850 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:49 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:31:49] "GET /metrics HTTP/1.1" 200 37924 "" "Prometheus/2.33.4" 2024-04-22T21:31:50.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-04-22T21:31:50.541 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:50 smithi130 ceph-mon[26394]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:50.541 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:50 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3062179119' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:31:50.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:50 smithi130 ceph-mon[21867]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:50.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:50 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3062179119' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:31:50.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:50 smithi150 ceph-mon[22676]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:50.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:50 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3062179119' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:31:52.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:52 smithi150 ceph-mon[22676]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:52.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:52 smithi130 ceph-mon[26394]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:52.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:52 smithi130 ceph-mon[21867]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:53.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:53 smithi130 ceph-mon[21867]: from='client.15471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:53.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:53 smithi150 ceph-mon[22676]: from='client.15471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:53.730 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:53 smithi130 ceph-mon[26394]: from='client.15471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:53.806 INFO:teuthology.orchestra.run.smithi130.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:31:54.443 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:54 smithi130 ceph-mon[21867]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:54.443 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:54 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:54.443 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:54 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:31:54.443 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:54 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:31:54.443 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:54 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:31:54.443 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:54 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:31:54.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:54 smithi130 ceph-mon[26394]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:54.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:54 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:54.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:54 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:31:54.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:54 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:31:54.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:54 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:31:54.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:54 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:31:54.467 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T21:31:54.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:54 smithi150 ceph-mon[22676]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:31:54.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:54 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:54.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:54 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:31:54.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:54 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:31:54.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:54 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:31:54.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:54 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:31:55.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:55 smithi130 ceph-mon[21867]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:31:55.591 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:55 smithi130 ceph-mon[21867]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:31:55.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:55 smithi130 ceph-mon[26394]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:31:55.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:55 smithi130 ceph-mon[26394]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:31:55.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:55 smithi150 ceph-mon[22676]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:31:55.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:55 smithi150 ceph-mon[22676]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:31:56.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[26394]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:56.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:56.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:31:56.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:31:56.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[26394]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-22T21:31:56.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-22T21:31:56.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[26394]: osdmap e106: 8 total, 8 up, 8 in 2024-04-22T21:31:56.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[26394]: Standby manager daemon x started 2024-04-22T21:31:56.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[21867]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:56.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:56.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:31:56.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:31:56.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[21867]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-22T21:31:56.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-22T21:31:56.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[21867]: osdmap e106: 8 total, 8 up, 8 in 2024-04-22T21:31:56.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-mon[21867]: Standby manager daemon x started 2024-04-22T21:31:56.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:56 smithi150 ceph-mon[22676]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:31:56.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:56 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:31:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:56 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' 2024-04-22T21:31:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:56 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:31:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:56 smithi150 ceph-mon[22676]: from='mgr.34943 172.21.15.150:0/2864081755' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-22T21:31:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:56 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-22T21:31:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:56 smithi150 ceph-mon[22676]: osdmap e106: 8 total, 8 up, 8 in 2024-04-22T21:31:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:56 smithi150 ceph-mon[22676]: Standby manager daemon x started 2024-04-22T21:31:56.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:31:56] ENGINE Bus STOPPING 2024-04-22T21:31:56.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:31:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T21:31:56.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:31:56] ENGINE Bus STOPPED 2024-04-22T21:31:56.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:31:56] ENGINE Bus STARTING 2024-04-22T21:31:56.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:31:56] ENGINE Serving on http://:::9283 2024-04-22T21:31:56.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:31:56 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:31:56] ENGINE Bus STARTED 2024-04-22T21:31:57.027 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:56 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:31:56.680+0000 7fbf612f8700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-22T21:31:57.027 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:56 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ignoring --setuser ceph since I am not root 2024-04-22T21:31:57.027 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:56 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ignoring --setgroup ceph since I am not root 2024-04-22T21:31:57.027 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:56 smithi150 ceph-mgr[42406]: -- 172.21.15.150:0/3855854925 <== mon.2 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559ecdcb45a0 con 0x559ecdc88c00 2024-04-22T21:31:57.027 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:56 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:31:56.855+0000 7f83e770e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:31:57.027 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:31:57.043+0000 7f83e770e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:31:57.329 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:31:57.150+0000 7f83e770e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:31:57.601 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:31:57.476+0000 7f83e770e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:31:57.601 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:31:57.571+0000 7f83e770e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:31:57.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:31:57.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:31:57.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-22T21:31:57.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-22T21:31:57.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: Failing over to other MGR 2024-04-22T21:31:57.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.34943 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-22T21:31:57.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: mgrmap e38: y(active, starting, since 0.994572s), standbys: x 2024-04-22T21:31:57.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:31:57.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:31:57.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:31:57.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:31:57.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:31:57.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:31:57.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:31:57.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:31:57.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:31:57.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:31:57.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:31:57.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:31:57.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:31:57.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:31:57.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:31:57.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:31:57.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: Manager daemon y is now available 2024-04-22T21:31:57.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:57.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:31:57.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:31:57.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:31:57.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:57 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:31:57.685 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:31:57.686 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:31:57.686 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:31:57.686 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-22T21:31:57.686 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-22T21:31:57.686 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: Failing over to other MGR 2024-04-22T21:31:57.687 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.34943 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-22T21:31:57.687 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: mgrmap e38: y(active, starting, since 0.994572s), standbys: x 2024-04-22T21:31:57.687 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:31:57.687 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:31:57.687 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:31:57.687 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:31:57.688 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:31:57.688 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:31:57.688 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:31:57.688 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:31:57.688 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:31:57.688 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:31:57.688 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:31:57.689 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:31:57.689 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:31:57.689 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:31:57.689 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:31:57.689 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:31:57.689 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: Manager daemon y is now available 2024-04-22T21:31:57.690 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:57.690 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:31:57.690 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:31:57.690 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:31:57.690 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:31:57.691 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:31:57.691 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:31:57.691 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-22T21:31:57.691 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-22T21:31:57.691 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: Failing over to other MGR 2024-04-22T21:31:57.692 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.34943 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-22T21:31:57.692 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: mgrmap e38: y(active, starting, since 0.994572s), standbys: x 2024-04-22T21:31:57.692 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:31:57.692 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:31:57.692 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:31:57.692 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:31:57.692 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:31:57.693 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:31:57.693 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:31:57.693 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:31:57.693 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:31:57.693 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:31:57.693 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:31:57.693 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:31:57.694 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:31:57.694 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:31:57.694 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:31:57.694 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:31:57.694 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: Manager daemon y is now available 2024-04-22T21:31:57.694 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:31:57.695 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:31:57.695 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:31:57.695 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:31:57.695 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:57 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:31:58.351 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:31:58.351 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (12m) 3m ago 20m 24.2M - 0.23.0 ba2b418f427c 23fd04f877ce 2024-04-22T21:31:58.351 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (12m) 3m ago 20m 50.2M - 8.3.5 dad864ee21e9 e021384d6729 2024-04-22T21:31:58.351 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (3m) 3m ago 19m 48.3M - 3.5 e1d6a67b021e 8cf00b4e4942 2024-04-22T21:31:58.351 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283 running (15m) 3m ago 25m 493M - 19.0.0-3198-g56e81df3 82086b19bb07 327605fe868f 2024-04-22T21:31:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (9m) 3m ago 26m 75.0M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:31:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (26m) 3m ago 27m 72.2M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:31:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (25m) 3m ago 25m 51.5M 2048M 17.2.0 e1d6a67b021e 861909a68cf9 2024-04-22T21:31:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (25m) 3m ago 25m 51.4M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:31:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (12m) 3m ago 21m 15.6M - 1.3.1 1dbe0e931976 4057e105dc28 2024-04-22T21:31:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (12m) 3m ago 21m 17.2M - 1.3.1 1dbe0e931976 2bcf91466257 2024-04-22T21:31:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (25m) 3m ago 25m 53.7M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:31:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (24m) 3m ago 24m 56.0M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:31:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (24m) 3m ago 24m 49.8M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:31:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (23m) 3m ago 23m 51.4M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:31:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (23m) 3m ago 23m 52.9M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:31:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (23m) 3m ago 23m 51.7M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:31:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (22m) 3m ago 22m 53.6M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:31:58.353 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (22m) 3m ago 22m 53.0M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:31:58.353 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (3m) 3m ago 21m 44.9M - 2.33.4 514e6a882f6e b7cd02579a7f 2024-04-22T21:31:58.353 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (19m) 3m ago 19m 76.3M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:31:58.353 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (19m) 3m ago 19m 74.0M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:31:58.580 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:58 smithi150 ceph-mon[22676]: [22/Apr/2024:21:31:57] ENGINE Bus STARTING 2024-04-22T21:31:58.581 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:58 smithi150 ceph-mon[22676]: [22/Apr/2024:21:31:57] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:31:58.581 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:58 smithi150 ceph-mon[22676]: [22/Apr/2024:21:31:57] ENGINE Serving on http://172.21.15.130:8765 2024-04-22T21:31:58.581 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:58 smithi150 ceph-mon[22676]: [22/Apr/2024:21:31:57] ENGINE Bus STARTED 2024-04-22T21:31:58.581 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:58 smithi150 ceph-mon[22676]: mgrmap e39: y(active, since 2s), standbys: x 2024-04-22T21:31:58.633 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:58 smithi130 ceph-mon[26394]: [22/Apr/2024:21:31:57] ENGINE Bus STARTING 2024-04-22T21:31:58.633 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:58 smithi130 ceph-mon[26394]: [22/Apr/2024:21:31:57] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:31:58.633 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:58 smithi130 ceph-mon[26394]: [22/Apr/2024:21:31:57] ENGINE Serving on http://172.21.15.130:8765 2024-04-22T21:31:58.633 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:58 smithi130 ceph-mon[26394]: [22/Apr/2024:21:31:57] ENGINE Bus STARTED 2024-04-22T21:31:58.634 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:58 smithi130 ceph-mon[26394]: mgrmap e39: y(active, since 2s), standbys: x 2024-04-22T21:31:58.634 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:58 smithi130 ceph-mon[21867]: [22/Apr/2024:21:31:57] ENGINE Bus STARTING 2024-04-22T21:31:58.634 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:58 smithi130 ceph-mon[21867]: [22/Apr/2024:21:31:57] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:31:58.634 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:58 smithi130 ceph-mon[21867]: [22/Apr/2024:21:31:57] ENGINE Serving on http://172.21.15.130:8765 2024-04-22T21:31:58.634 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:58 smithi130 ceph-mon[21867]: [22/Apr/2024:21:31:57] ENGINE Bus STARTED 2024-04-22T21:31:58.634 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:58 smithi130 ceph-mon[21867]: mgrmap e39: y(active, since 2s), standbys: x 2024-04-22T21:31:58.791 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:31:58.791 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:31:58.791 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T21:31:58.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:31:58.791 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:31:58.791 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:31:58.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:31:58.792 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:31:58.792 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:31:58.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:31:58.792 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": {}, 2024-04-22T21:31:58.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:31:58.792 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:31:58.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:31:58.792 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:31:58.792 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T21:31:58.792 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:31:58.792 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:31:58.792 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:31:58.850 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:31:58.835+0000 7f83e770e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:31:59.128 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:31:59.128 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:31:59.128 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:31:59.128 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-22T21:31:59.128 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-04-22T21:31:59.128 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-04-22T21:31:59.128 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-04-22T21:31:59.128 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "2/2 daemons upgraded", 2024-04-22T21:31:59.128 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "", 2024-04-22T21:31:59.129 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:31:59.129 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:31:59.334 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:31:59.012+0000 7f83e770e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:31:59.334 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:31:59.221+0000 7f83e770e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:31:59.600 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:31:59.534+0000 7f83e770e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:31:59.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:59 smithi150 ceph-mon[22676]: from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:59.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:59 smithi150 ceph-mon[22676]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:31:59.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:59 smithi150 ceph-mon[22676]: from='client.15507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:59.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:59 smithi150 ceph-mon[22676]: from='client.15513 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:59.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:59 smithi150 ceph-mon[22676]: mgrmap e40: y(active, since 3s), standbys: x 2024-04-22T21:31:59.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:31:59 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3131134445' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:31:59.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:59 smithi130 ceph-mon[21867]: from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:59.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:59 smithi130 ceph-mon[21867]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:31:59.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:59 smithi130 ceph-mon[21867]: from='client.15507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:59.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:59 smithi130 ceph-mon[21867]: from='client.15513 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:59.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:59 smithi130 ceph-mon[21867]: mgrmap e40: y(active, since 3s), standbys: x 2024-04-22T21:31:59.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:31:59 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3131134445' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:31:59.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:59 smithi130 ceph-mon[26394]: from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:59.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:59 smithi130 ceph-mon[26394]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:31:59.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:59 smithi130 ceph-mon[26394]: from='client.15507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:59.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:59 smithi130 ceph-mon[26394]: from='client.15513 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:31:59.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:59 smithi130 ceph-mon[26394]: mgrmap e40: y(active, since 3s), standbys: x 2024-04-22T21:31:59.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:31:59 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3131134445' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:00.059 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:31:59.629+0000 7f83e770e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:32:00.059 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:31:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:31:59.841+0000 7f83e770e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:32:00.059 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:00 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:00.076+0000 7f83e770e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:32:00.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:00 smithi130 ceph-mon[26394]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:00.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:00 smithi130 ceph-mon[26394]: from='client.25414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:00.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:00 smithi130 ceph-mon[21867]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:00.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:00 smithi130 ceph-mon[21867]: from='client.25414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:00.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:00 smithi150 ceph-mon[22676]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:00.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:00 smithi150 ceph-mon[22676]: from='client.25414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:01.023 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:00 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:00.697+0000 7f83e770e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:32:01.291 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:01 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:01.160+0000 7f83e770e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:32:01.291 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:01 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:01.261+0000 7f83e770e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:32:01.600 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:01 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:01.468+0000 7f83e770e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T21:32:02.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:01 smithi150 ceph-mon[22676]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:02.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:01 smithi150 ceph-mon[22676]: mgrmap e41: y(active, since 5s), standbys: x 2024-04-22T21:32:02.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:01 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:02.025 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:01 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:02.025 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:01 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:02.025 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:01 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:02.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:01 smithi130 ceph-mon[26394]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:02.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:01 smithi130 ceph-mon[26394]: mgrmap e41: y(active, since 5s), standbys: x 2024-04-22T21:32:02.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:01 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:02.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:01 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:02.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:01 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:02.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:01 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:02.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:01 smithi130 ceph-mon[21867]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:02.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:01 smithi130 ceph-mon[21867]: mgrmap e41: y(active, since 5s), standbys: x 2024-04-22T21:32:02.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:01 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:02.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:01 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:02.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:01 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:02.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:01 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:02.350 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:02.041+0000 7f83e770e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:32:02.350 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:02.140+0000 7f83e770e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:32:02.765 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:02.403+0000 7f83e770e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:32:02.765 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:02.548+0000 7f83e770e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:32:02.765 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:02.653+0000 7f83e770e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:32:02.766 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:02.782+0000 7f83e770e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:32:03.350 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:03.106+0000 7f83e770e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:32:03.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:03 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:03.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:03 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:03.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:03 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:03.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:03 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:03.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:03 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:03.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:03 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:32:03.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:03 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:03.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:03 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:03.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:03 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:03.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:03 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:03.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:03 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:03.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:03 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:32:03.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:03 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:03.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:03 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:03.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:03 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:03.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:03 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:03.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:03 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:03.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:03 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:32:03.851 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:03.764+0000 7f83e770e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:32:04.350 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:03.993+0000 7f83e770e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:32:04.351 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:32:03] ENGINE Bus STARTING 2024-04-22T21:32:04.351 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: CherryPy Checker: 2024-04-22T21:32:04.351 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: The Application mounted at '' has an empty config. 2024-04-22T21:32:04.351 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: 2024-04-22T21:32:04.351 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:32:04] ENGINE Serving on http://:::9283 2024-04-22T21:32:04.351 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: [22/Apr/2024:21:32:04] ENGINE Bus STARTED 2024-04-22T21:32:04.746 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:04.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:32:04.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:32:04.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:32:04.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:32:04.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:32:04.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:32:04.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:32:04.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:32:04.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.748 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:04.748 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:04.748 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:04.748 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: Standby manager daemon x restarted 2024-04-22T21:32:04.748 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: Standby manager daemon x started 2024-04-22T21:32:04.748 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/3452833411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:32:04.748 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/3452833411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:32:04.748 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/3452833411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:32:04.748 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/3452833411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:32:04.748 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:04.748 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:32:04.748 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:04.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:04.750 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: Standby manager daemon x restarted 2024-04-22T21:32:04.750 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: Standby manager daemon x started 2024-04-22T21:32:04.750 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/3452833411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:32:04.750 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/3452833411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:32:04.750 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/3452833411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:32:04.750 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:04 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/3452833411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:32:04.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:04.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:32:04.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:32:04.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:32:04.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:32:04.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:32:04.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:32:04.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:32:04.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:32:04.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: Standby manager daemon x restarted 2024-04-22T21:32:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: Standby manager daemon x started 2024-04-22T21:32:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/3452833411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:32:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/3452833411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:32:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/3452833411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:32:04.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:04 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/3452833411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:32:05.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:05 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:32:05.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:05 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:32:05.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:05 smithi130 ceph-mon[21867]: mgrmap e42: y(active, since 9s), standbys: x 2024-04-22T21:32:05.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:05 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:32:05.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:05 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:32:05.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:05 smithi130 ceph-mon[26394]: mgrmap e42: y(active, since 9s), standbys: x 2024-04-22T21:32:05.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:05 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:32:05.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:05 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:32:05.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:05 smithi150 ceph-mon[22676]: mgrmap e42: y(active, since 9s), standbys: x 2024-04-22T21:32:06.736 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:06 smithi130 ceph-mon[21867]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:32:06.736 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:06 smithi130 ceph-mon[21867]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:32:06.736 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:06 smithi130 ceph-mon[26394]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:32:06.737 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:06 smithi130 ceph-mon[26394]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:32:06.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:06 smithi150 ceph-mon[22676]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:32:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:06 smithi150 ceph-mon[22676]: from='client.15375 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:32:07.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:32:07.214+0000 7fdddea21700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:07.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:32:07.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:07.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:07.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:07.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:07.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:07.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:32:07.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:07.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:32:07.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:07.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:32:07.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:07.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:07.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:07.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:07.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:07.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:07.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:07.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:07.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:07.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:32:07.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:32:07.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:32:07.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:32:07.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:32:07.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:32:07.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:32:07.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:32:07.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return future.result(timeout) 2024-04-22T21:32:07.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:32:07.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.__get_result() 2024-04-22T21:32:07.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: raise self._exception 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: use_current_daemon_image=reconfig, 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:07.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:32:07.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:07.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:32:07.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:07.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:07.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:07.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:07.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:07.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:07.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:07.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:07.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:07 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:08.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:08 smithi150 ceph-mon[22676]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:32:08.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:08 smithi150 ceph-mon[22676]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:08.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:08.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:08.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:08.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:08.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:08.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:08.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:32:08.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:08.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:32:08.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: return future.result(timeout) 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.__get_result() 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: raise self._exception 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:32:08.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:08.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:08.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:08.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:08.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:08.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:08.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:08.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:08.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:08 smithi150 ceph-mon[22676]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:32:08.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:08 smithi150 ceph-mon[22676]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:32:08.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:08 smithi130 ceph-mon[26394]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:32:08.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:08 smithi130 ceph-mon[26394]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:08.797 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:08.797 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:08.797 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:08.797 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:32:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:08.798 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:32:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:32:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:32:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:32:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:32:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:32:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:32:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: return future.result(timeout) 2024-04-22T21:32:08.799 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.__get_result() 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: raise self._exception 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:08.800 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:08 smithi130 ceph-mon[26394]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:32:08.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:08 smithi130 ceph-mon[26394]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:32:08.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:08 smithi130 ceph-mon[21867]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:32:08.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:08 smithi130 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:08.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:08.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:08.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:08.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:08.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:08.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:08.802 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: return future.result(timeout) 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.__get_result() 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: raise self._exception 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:32:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:32:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:08 smithi130 ceph-mon[21867]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T21:32:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:08 smithi130 ceph-mon[21867]: Reconfiguring daemon prometheus.a on smithi150 2024-04-22T21:32:09.764 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[42402]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:32:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T21:32:10.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:10 smithi150 ceph-mon[22676]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-04-22T21:32:10.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:10 smithi130 ceph-mon[26394]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-04-22T21:32:10.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:10 smithi130 ceph-mon[21867]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-04-22T21:32:10.997 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:10 smithi150 systemd[1]: Stopping Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:32:10.998 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:10 smithi150 bash[57380]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T21:32:11.289 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:32:11.014Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T21:32:11.290 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:32:11.014Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T21:32:11.290 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:32:11.014Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T21:32:11.290 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:32:11.014Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T21:32:11.290 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:32:11.014Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T21:32:11.290 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:32:11.014Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T21:32:11.290 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:32:11.015Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T21:32:11.290 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:32:11.015Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T21:32:11.290 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:32:11.015Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T21:32:11.290 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:32:11.015Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T21:32:11.290 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:32:11.015Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T21:32:11.290 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[54585]: ts=2024-04-22T21:32:11.015Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T21:32:11.290 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 podman[57393]: 2024-04-22 21:32:11.024786218 +0000 UTC m=+0.166742565 container died b7cd02579a7ff838594445bec5be9b78d92d5a9760c8062042e05d2aa234f294 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:32:11.600 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 podman[57393]: 2024-04-22 21:32:11.422490849 +0000 UTC m=+0.564447168 container cleanup b7cd02579a7ff838594445bec5be9b78d92d5a9760c8062042e05d2aa234f294 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:32:11.601 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 bash[57393]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T21:32:11.601 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 bash[57420]: Error: no container with name or ID "ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T21:32:11.890 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 podman[57407]: 2024-04-22 21:32:11.648665396 +0000 UTC m=+0.623332686 container remove b7cd02579a7ff838594445bec5be9b78d92d5a9760c8062042e05d2aa234f294 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:32:11.891 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T21:32:11.891 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 systemd[1]: Stopped Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:32:11.891 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.404s CPU time. 2024-04-22T21:32:11.891 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 systemd[1]: Starting Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:32:11.891 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 podman[57493]: 2024-04-22 21:32:11.788285306 +0000 UTC m=+0.018857959 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T21:32:11.891 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:11 smithi150 podman[57493]: 2024-04-22 21:32:11.907911114 +0000 UTC m=+0.138483803 container create bcd947b380404de8071f92c34445d2f728dc4fd888ad0a84b9e181918e299bbd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:32:12.161 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 podman[57493]: 2024-04-22 21:32:12.151781698 +0000 UTC m=+0.382354342 container init bcd947b380404de8071f92c34445d2f728dc4fd888ad0a84b9e181918e299bbd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:32:12.161 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 podman[57493]: 2024-04-22 21:32:12.154280698 +0000 UTC m=+0.384853342 container start bcd947b380404de8071f92c34445d2f728dc4fd888ad0a84b9e181918e299bbd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:32:12.544 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.178Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T21:32:12.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.178Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T21:32:12.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.178Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T21:32:12.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.178Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi150 (none))" 2024-04-22T21:32:12.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.178Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T21:32:12.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.178Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T21:32:12.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.179Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T21:32:12.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.180Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T21:32:12.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.180Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T21:32:12.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.183Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T21:32:12.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.183Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.538µs 2024-04-22T21:32:12.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.183Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T21:32:12.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 bash[57493]: bcd947b380404de8071f92c34445d2f728dc4fd888ad0a84b9e181918e299bbd 2024-04-22T21:32:12.545 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 systemd[1]: Started Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:32:12.546 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.365Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-04-22T21:32:12.795 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.605Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-04-22T21:32:12.795 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-mon[22676]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-04-22T21:32:12.795 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:32:12.795 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:12.795 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:12.795 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:32:12.795 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T21:32:12.796 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:12.796 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:32:12.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[21867]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-04-22T21:32:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:32:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:32:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T21:32:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:32:12.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:32:12] ENGINE Bus STOPPING 2024-04-22T21:32:12.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:32:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T21:32:12.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:32:12] ENGINE Bus STOPPED 2024-04-22T21:32:12.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:32:12] ENGINE Bus STARTING 2024-04-22T21:32:12.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:32:12] ENGINE Serving on http://:::9283 2024-04-22T21:32:12.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:32:12] ENGINE Bus STARTED 2024-04-22T21:32:12.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[26394]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-04-22T21:32:12.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:32:12.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:12.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:12.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:32:12.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T21:32:12.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:12.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:12 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:32:13.070 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.812Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-04-22T21:32:13.071 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.815Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-04-22T21:32:13.071 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.817Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-04-22T21:32:13.071 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.817Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-04-22T21:32:13.071 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.817Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=44.413µs wal_replay_duration=634.024601ms total_replay_duration=634.081923ms 2024-04-22T21:32:13.071 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.822Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T21:32:13.071 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.822Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T21:32:13.071 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.822Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T21:32:13.071 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.845Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.478089ms db_storage=991ns remote_storage=1.107µs web_handler=725ns query_engine=906ns scrape=243.705µs scrape_sd=104.817µs notify=18.799µs notify_sd=19.262µs rules=21.749698ms 2024-04-22T21:32:13.071 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:32:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:32:12.845Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T21:32:13.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:13 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:32:13.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:13 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T21:32:14.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:13 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:32:14.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:13 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T21:32:14.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:13 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:32:14.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:13 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T21:32:14.811 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:14 smithi150 ceph-mon[22676]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-04-22T21:32:14.811 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:14 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:32:14.811 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:14 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:32:14.811 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:14 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:32:14.811 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:14 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:15.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:14 smithi130 ceph-mon[21867]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-04-22T21:32:15.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:14 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:32:15.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:14 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:32:15.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:14 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:32:15.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:14 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:15.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:14 smithi130 ceph-mon[26394]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-04-22T21:32:15.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:14 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:32:15.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:14 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:32:15.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:14 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:32:15.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:14 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:15.799 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:15 smithi150 ceph-mon[22676]: Upgrade: Updating mgr.x 2024-04-22T21:32:15.823 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:15 smithi150 ceph-mon[22676]: Deploying daemon mgr.x on smithi150 2024-04-22T21:32:16.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:15 smithi130 ceph-mon[21867]: Upgrade: Updating mgr.x 2024-04-22T21:32:16.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:15 smithi130 ceph-mon[21867]: Deploying daemon mgr.x on smithi150 2024-04-22T21:32:16.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:15 smithi130 ceph-mon[26394]: Upgrade: Updating mgr.x 2024-04-22T21:32:16.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:15 smithi130 ceph-mon[26394]: Deploying daemon mgr.x on smithi150 2024-04-22T21:32:16.100 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:15 smithi150 systemd[1]: Stopping Ceph mgr.x for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:32:16.403 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:16 smithi150 podman[57843]: 2024-04-22 21:32:16.150983535 +0000 UTC m=+0.191475816 container died 327605fe868fbd6f31a6a1d7799d6f873fbb9b77ca5f0915b277c808e6ee11eb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-22T21:32:16.769 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:16 smithi150 ceph-mon[22676]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-04-22T21:32:16.770 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:16 smithi150 podman[57843]: 2024-04-22 21:32:16.502610382 +0000 UTC m=+0.543102672 container cleanup 327605fe868fbd6f31a6a1d7799d6f873fbb9b77ca5f0915b277c808e6ee11eb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, RELEASE=main-56e81df, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416) 2024-04-22T21:32:16.770 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:16 smithi150 bash[57843]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x 2024-04-22T21:32:17.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:16 smithi130 ceph-mon[26394]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-04-22T21:32:17.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:16 smithi130 ceph-mon[21867]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 129 op/s 2024-04-22T21:32:17.096 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:16 smithi150 podman[57858]: 2024-04-22 21:32:16.787056174 +0000 UTC m=+0.635249331 container remove 327605fe868fbd6f31a6a1d7799d6f873fbb9b77ca5f0915b277c808e6ee11eb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-56e81df, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-22T21:32:17.096 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:16 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-22T21:32:17.097 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:16 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-22T21:32:17.097 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:16 smithi150 systemd[1]: Stopped Ceph mgr.x for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:32:17.097 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:16 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.x.service: Consumed 58.678s CPU time. 2024-04-22T21:32:17.097 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:17 smithi150 systemd[1]: Starting Ceph mgr.x for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:32:17.350 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:17 smithi150 podman[57994]: 2024-04-22 21:32:17.232640504 +0000 UTC m=+0.018266077 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:32:17.350 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:17 smithi150 podman[57994]: 2024-04-22 21:32:17.329854433 +0000 UTC m=+0.115479980 container create db4c3cf522640658cd9cd07fe8fc26d758e3f288b94dc7a957d5a98675ed17b8 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-56e81df) 2024-04-22T21:32:17.699 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:17 smithi150 podman[57994]: 2024-04-22 21:32:17.565811706 +0000 UTC m=+0.351437251 container init db4c3cf522640658cd9cd07fe8fc26d758e3f288b94dc7a957d5a98675ed17b8 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, RELEASE=main-56e81df) 2024-04-22T21:32:17.699 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:17 smithi150 podman[57994]: 2024-04-22 21:32:17.568722415 +0000 UTC m=+0.354347963 container start db4c3cf522640658cd9cd07fe8fc26d758e3f288b94dc7a957d5a98675ed17b8 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-56e81df) 2024-04-22T21:32:17.699 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:17 smithi150 ceph-mgr[58012]: -- 172.21.15.150:0/154648894 <== mon.2 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559204eb05a0 con 0x559204e84800 2024-04-22T21:32:18.013 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:17 smithi150 bash[57994]: db4c3cf522640658cd9cd07fe8fc26d758e3f288b94dc7a957d5a98675ed17b8 2024-04-22T21:32:18.013 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:17 smithi150 systemd[1]: Started Ceph mgr.x for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:32:18.013 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:17 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:17.721+0000 7f19e99f4200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:32:18.013 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:17 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:17.920+0000 7f19e99f4200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:32:18.013 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:18.028+0000 7f19e99f4200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:32:18.764 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:18 smithi150 ceph-mon[22676]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T21:32:18.764 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:18 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:18.764 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:18 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:18.764 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:18.380+0000 7f19e99f4200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:32:18.764 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:18.473+0000 7f19e99f4200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:32:18.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:18 smithi130 ceph-mon[21867]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T21:32:18.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:18.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:18 smithi130 ceph-mon[26394]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T21:32:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:19.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:32:18] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-04-22T21:32:19.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:19 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2019356370' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:32:19.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:19 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4107365076' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2277881448"}]: dispatch 2024-04-22T21:32:19.912 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:19.741+0000 7f19e99f4200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:32:19.913 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:19.928+0000 7f19e99f4200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:32:20.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:19 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2019356370' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:32:20.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:19 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4107365076' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2277881448"}]: dispatch 2024-04-22T21:32:20.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:19 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2019356370' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:32:20.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:19 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4107365076' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2277881448"}]: dispatch 2024-04-22T21:32:20.303 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:20.121+0000 7f19e99f4200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:32:20.568 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:20.439+0000 7f19e99f4200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:32:20.568 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:20.535+0000 7f19e99f4200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:32:20.850 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:20.746+0000 7f19e99f4200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:32:20.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:20 smithi150 ceph-mon[22676]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T21:32:20.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:20 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4107365076' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2277881448"}]': finished 2024-04-22T21:32:20.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:20 smithi150 ceph-mon[22676]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T21:32:20.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:20 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3242463714' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2314619228"}]: dispatch 2024-04-22T21:32:20.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:20 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2314619228"}]: dispatch 2024-04-22T21:32:21.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:20 smithi130 ceph-mon[26394]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T21:32:21.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:20 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4107365076' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2277881448"}]': finished 2024-04-22T21:32:21.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:20 smithi130 ceph-mon[26394]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T21:32:21.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:20 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3242463714' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2314619228"}]: dispatch 2024-04-22T21:32:21.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:20 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2314619228"}]: dispatch 2024-04-22T21:32:21.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:20 smithi130 ceph-mon[21867]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T21:32:21.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:20 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4107365076' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2277881448"}]': finished 2024-04-22T21:32:21.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:20 smithi130 ceph-mon[21867]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T21:32:21.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:20 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3242463714' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2314619228"}]: dispatch 2024-04-22T21:32:21.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:20 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2314619228"}]: dispatch 2024-04-22T21:32:21.272 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:20.959+0000 7f19e99f4200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:32:21.850 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:21 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:21.585+0000 7f19e99f4200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:32:21.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:21 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2314619228"}]': finished 2024-04-22T21:32:21.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:21 smithi150 ceph-mon[22676]: osdmap e108: 8 total, 8 up, 8 in 2024-04-22T21:32:21.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:21 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1974185655' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1358585731"}]: dispatch 2024-04-22T21:32:21.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:21 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1358585731"}]: dispatch 2024-04-22T21:32:22.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:21 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2314619228"}]': finished 2024-04-22T21:32:22.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:21 smithi130 ceph-mon[26394]: osdmap e108: 8 total, 8 up, 8 in 2024-04-22T21:32:22.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:21 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1974185655' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1358585731"}]: dispatch 2024-04-22T21:32:22.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:21 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1358585731"}]: dispatch 2024-04-22T21:32:22.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:21 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2314619228"}]': finished 2024-04-22T21:32:22.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:21 smithi130 ceph-mon[21867]: osdmap e108: 8 total, 8 up, 8 in 2024-04-22T21:32:22.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:21 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1974185655' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1358585731"}]: dispatch 2024-04-22T21:32:22.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:21 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1358585731"}]: dispatch 2024-04-22T21:32:22.339 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:22.035+0000 7f19e99f4200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:32:22.339 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:22.135+0000 7f19e99f4200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:32:22.339 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:22.355+0000 7f19e99f4200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T21:32:22.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:22 smithi150 ceph-mon[22676]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:22.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:22 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1358585731"}]': finished 2024-04-22T21:32:22.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:22 smithi150 ceph-mon[22676]: osdmap e109: 8 total, 8 up, 8 in 2024-04-22T21:32:22.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:22 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3855131673' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/246667944"}]: dispatch 2024-04-22T21:32:22.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:22 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/246667944"}]: dispatch 2024-04-22T21:32:22.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:22 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:22.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:22 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:22.947 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[21867]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:22.948 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1358585731"}]': finished 2024-04-22T21:32:22.948 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[21867]: osdmap e109: 8 total, 8 up, 8 in 2024-04-22T21:32:22.948 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3855131673' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/246667944"}]: dispatch 2024-04-22T21:32:22.948 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/246667944"}]: dispatch 2024-04-22T21:32:22.948 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:22.948 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:22.948 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[26394]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:22.948 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/1358585731"}]': finished 2024-04-22T21:32:22.948 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[26394]: osdmap e109: 8 total, 8 up, 8 in 2024-04-22T21:32:22.948 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3855131673' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/246667944"}]: dispatch 2024-04-22T21:32:22.949 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/246667944"}]: dispatch 2024-04-22T21:32:22.949 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:22.949 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:22 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:23.265 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:22.918+0000 7f19e99f4200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:32:23.265 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:23.016+0000 7f19e99f4200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:32:23.516 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:23.281+0000 7f19e99f4200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:32:23.516 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:23.427+0000 7f19e99f4200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:32:23.516 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:23.531+0000 7f19e99f4200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:32:23.850 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:23.660+0000 7f19e99f4200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:32:23.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:23 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/246667944"}]': finished 2024-04-22T21:32:23.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:23 smithi150 ceph-mon[22676]: osdmap e110: 8 total, 8 up, 8 in 2024-04-22T21:32:23.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:23 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2380799589' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/246667944"}]: dispatch 2024-04-22T21:32:23.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:23 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/246667944"}]: dispatch 2024-04-22T21:32:23.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:23 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:23.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:23 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:24.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:23 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/246667944"}]': finished 2024-04-22T21:32:24.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:23 smithi130 ceph-mon[21867]: osdmap e110: 8 total, 8 up, 8 in 2024-04-22T21:32:24.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:23 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2380799589' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/246667944"}]: dispatch 2024-04-22T21:32:24.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:23 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/246667944"}]: dispatch 2024-04-22T21:32:24.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:23 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:24.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:23 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:24.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:23 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/246667944"}]': finished 2024-04-22T21:32:24.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:23 smithi130 ceph-mon[26394]: osdmap e110: 8 total, 8 up, 8 in 2024-04-22T21:32:24.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:23 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2380799589' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/246667944"}]: dispatch 2024-04-22T21:32:24.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:23 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/246667944"}]: dispatch 2024-04-22T21:32:24.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:23 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:24.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:23 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:24.224 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:23.985+0000 7f19e99f4200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:32:24.938 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:24 smithi150 ceph-mon[22676]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:24.939 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:24 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/246667944"}]': finished 2024-04-22T21:32:24.939 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:24 smithi150 ceph-mon[22676]: osdmap e111: 8 total, 8 up, 8 in 2024-04-22T21:32:24.939 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:24 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2113021745' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2409199828"}]: dispatch 2024-04-22T21:32:24.939 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:24 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2409199828"}]: dispatch 2024-04-22T21:32:24.939 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:24.619+0000 7f19e99f4200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:32:24.939 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:24.847+0000 7f19e99f4200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:32:24.939 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: [22/Apr/2024:21:32:24] ENGINE Bus STARTING 2024-04-22T21:32:24.939 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: CherryPy Checker: 2024-04-22T21:32:24.940 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: The Application mounted at '' has an empty config. 2024-04-22T21:32:24.940 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:32:24.940 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: [22/Apr/2024:21:32:24] ENGINE Serving on http://:::9283 2024-04-22T21:32:24.940 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:32:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: [22/Apr/2024:21:32:24] ENGINE Bus STARTED 2024-04-22T21:32:25.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:24 smithi130 ceph-mon[26394]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:25.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:24 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/246667944"}]': finished 2024-04-22T21:32:25.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:24 smithi130 ceph-mon[26394]: osdmap e111: 8 total, 8 up, 8 in 2024-04-22T21:32:25.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:24 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2113021745' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2409199828"}]: dispatch 2024-04-22T21:32:25.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:24 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2409199828"}]: dispatch 2024-04-22T21:32:25.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:24 smithi130 ceph-mon[21867]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:32:25.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:24 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/246667944"}]': finished 2024-04-22T21:32:25.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:24 smithi130 ceph-mon[21867]: osdmap e111: 8 total, 8 up, 8 in 2024-04-22T21:32:25.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:24 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2113021745' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2409199828"}]: dispatch 2024-04-22T21:32:25.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:24 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2409199828"}]: dispatch 2024-04-22T21:32:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:25 smithi150 ceph-mon[22676]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2409199828"}]': finished 2024-04-22T21:32:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:25 smithi150 ceph-mon[22676]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T21:32:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:25 smithi150 ceph-mon[22676]: Standby manager daemon x restarted 2024-04-22T21:32:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:25 smithi150 ceph-mon[22676]: Standby manager daemon x started 2024-04-22T21:32:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:25 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/1779456421' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:32:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:25 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/1779456421' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:32:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:25 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/1779456421' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:32:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:25 smithi150 ceph-mon[22676]: from='mgr.? 172.21.15.150:0/1779456421' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:32:26.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[26394]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2409199828"}]': finished 2024-04-22T21:32:26.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[26394]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T21:32:26.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[26394]: Standby manager daemon x restarted 2024-04-22T21:32:26.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[26394]: Standby manager daemon x started 2024-04-22T21:32:26.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/1779456421' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:32:26.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/1779456421' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:32:26.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/1779456421' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:32:26.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/1779456421' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:32:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[21867]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2409199828"}]': finished 2024-04-22T21:32:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[21867]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T21:32:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[21867]: Standby manager daemon x restarted 2024-04-22T21:32:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[21867]: Standby manager daemon x started 2024-04-22T21:32:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/1779456421' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:32:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/1779456421' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:32:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/1779456421' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:32:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:25 smithi130 ceph-mon[21867]: from='mgr.? 172.21.15.150:0/1779456421' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:32:26.791 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:26 smithi150 ceph-mon[22676]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:32:26.791 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:26 smithi150 ceph-mon[22676]: mgrmap e43: y(active, since 29s), standbys: x 2024-04-22T21:32:27.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:26 smithi130 ceph-mon[26394]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:32:27.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:26 smithi130 ceph-mon[26394]: mgrmap e43: y(active, since 29s), standbys: x 2024-04-22T21:32:27.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:26 smithi130 ceph-mon[21867]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:32:27.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:26 smithi130 ceph-mon[21867]: mgrmap e43: y(active, since 29s), standbys: x 2024-04-22T21:32:27.872 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:32:27.872 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:27.872 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:27.872 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:27.872 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:32:27.872 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:27.872 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:27.872 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:27.872 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:27.872 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:32:27.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:27.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:27.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:27.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:32:27.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:27.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:27.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:27.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:27 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:28.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:27 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:32:28.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:27 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:28.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:27 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:28.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:27 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:28.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:27 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:32:28.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:27 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:28.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:27 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:28.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:27 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:28.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:27 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:28.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:28 smithi130 ceph-mon[21867]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 1 op/s 2024-04-22T21:32:28.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:28 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:32:28.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:28 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:32:28.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:28 smithi130 ceph-mon[26394]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 1 op/s 2024-04-22T21:32:28.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:28 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:32:28.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:28 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:32:29.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:28 smithi150 ceph-mon[22676]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 1 op/s 2024-04-22T21:32:29.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:28 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:32:29.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:28 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:32:29.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:32:28] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.33.4" 2024-04-22T21:32:29.490 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:32:29.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:29 smithi130 ceph-mon[21867]: from='client.35180 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:32:29.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:29 smithi130 ceph-mon[21867]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T21:32:29.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:29 smithi130 ceph-mon[21867]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:29.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:29 smithi130 ceph-mon[26394]: from='client.35180 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:32:29.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:29 smithi130 ceph-mon[26394]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T21:32:29.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:29 smithi130 ceph-mon[26394]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:30.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:29 smithi150 ceph-mon[22676]: from='client.35180 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:32:30.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:29 smithi150 ceph-mon[22676]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T21:32:30.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:29 smithi150 ceph-mon[22676]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:30.188 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:32:30.188 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (13m) 28s ago 21m 24.0M - 0.23.0 ba2b418f427c 23fd04f877ce 2024-04-22T21:32:30.188 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (13m) 7s ago 20m 50.9M - 8.3.5 dad864ee21e9 e021384d6729 2024-04-22T21:32:30.188 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (4m) 28s ago 20m 51.6M - 3.5 e1d6a67b021e 8cf00b4e4942 2024-04-22T21:32:30.188 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (12s) 7s ago 25m 67.5M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (9m) 28s ago 27m 477M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (27m) 28s ago 27m 74.9M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (26m) 7s ago 26m 54.7M 2048M 17.2.0 e1d6a67b021e 861909a68cf9 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (26m) 28s ago 26m 51.6M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (13m) 28s ago 21m 15.6M - 1.3.1 1dbe0e931976 4057e105dc28 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (13m) 7s ago 21m 19.5M - 1.3.1 1dbe0e931976 2bcf91466257 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (25m) 28s ago 25m 54.0M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (25m) 28s ago 25m 56.1M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (24m) 28s ago 24m 49.9M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (24m) 28s ago 24m 51.5M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (23m) 7s ago 23m 55.0M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (23m) 7s ago 23m 52.5M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (23m) 7s ago 23m 54.8M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:32:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (22m) 7s ago 22m 54.4M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:32:30.190 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (18s) 7s ago 22m 54.2M - 2.33.4 514e6a882f6e bcd947b38040 2024-04-22T21:32:30.190 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (20m) 28s ago 20m 77.5M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:32:30.190 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (20m) 7s ago 20m 74.9M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:32:30.547 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:32:30.170+0000 7fdddea21700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:30.547 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:32:30.547 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:30.547 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:30.547 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:30.548 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:30.549 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:30.549 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:30.549 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:32:30.549 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:32:30.549 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:32:30.549 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:32:30.549 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:32:30.549 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:32:30.549 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return future.result(timeout) 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.__get_result() 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: raise self._exception 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: use_current_daemon_image=reconfig, 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:30.550 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:30.551 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:30.552 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:30.552 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:30.620 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:32:30.620 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:32:30.620 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T21:32:30.620 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:32:30.621 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:32:30.621 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:32:30.621 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:32:30.621 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:32:30.621 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:32:30.621 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:32:30.621 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": {}, 2024-04-22T21:32:30.621 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:32:30.621 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:32:30.621 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:32:30.621 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:32:30.621 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T21:32:30.621 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:32:30.622 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:32:30.622 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:32:30.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:30.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:30.893 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:30.893 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:30.893 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:30.893 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:30.893 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:30.893 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:30.893 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:30.894 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:30.894 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:30.894 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:30.894 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:32:30.894 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:30.894 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:30.894 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:30.894 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:30.894 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:30.894 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:30.894 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:30.894 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:30.894 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: return future.result(timeout) 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.__get_result() 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: raise self._exception 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:32:30.895 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:32:30.896 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:30.897 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:30.897 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:30.897 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:30.897 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:30.897 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:30.897 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:30.897 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:30.897 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:30.897 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:30.897 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:30.898 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: return future.result(timeout) 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.__get_result() 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: raise self._exception 2024-04-22T21:32:30.899 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:30.900 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:32:30.901 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:30.901 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:30.901 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:30.901 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:30.901 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:30.901 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:30.901 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:30.901 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all mgr 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all crash 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all mds 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all nfs 2024-04-22T21:32:30.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:32:30.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3649642639' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.904 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:30.904 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:30.904 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all mgr 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all crash 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all mds 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.907 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all nfs 2024-04-22T21:32:30.907 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.907 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.907 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:32:30.907 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:30.907 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:30 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3649642639' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:30.976 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:32:30.976 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:32:30.976 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:32:30.976 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-22T21:32:30.976 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-04-22T21:32:30.977 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-04-22T21:32:30.977 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-04-22T21:32:30.977 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "2/2 daemons upgraded", 2024-04-22T21:32:30.977 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading node-exporter daemons", 2024-04-22T21:32:30.977 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:32:30.977 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:32:31.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:31.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:31.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:31.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:31.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:31.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:31.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:31.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:31.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:32:31.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:31.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:31.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:31.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:32:31.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:31.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:31.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:31.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:31.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:31.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:31.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:31.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:31.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:31.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:32:31.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:32:31.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:32:31.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: return future.result(timeout) 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.__get_result() 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: raise self._exception 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:31.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:31.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:32:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: Upgrade: Setting container_image for all mgr 2024-04-22T21:32:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-22T21:32:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-22T21:32:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='client.15615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-22T21:32:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-22T21:32:31.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-22T21:32:31.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-22T21:32:31.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:31.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: Upgrade: Setting container_image for all crash 2024-04-22T21:32:31.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:31.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:31.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: Upgrade: Setting container_image for all mds 2024-04-22T21:32:31.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:31.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:31.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:32:31.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:31.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:31.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T21:32:31.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:31.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:31.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:32:31.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:31.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:31.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: Upgrade: Setting container_image for all nfs 2024-04-22T21:32:31.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:31.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:31.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:32:31.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:31.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:30 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3649642639' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:31.896 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:31 smithi130 ceph-mon[21867]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:32:31.896 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:31 smithi130 ceph-mon[21867]: from='client.15627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:31.896 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:31 smithi130 ceph-mon[26394]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:32:31.897 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:31 smithi130 ceph-mon[26394]: from='client.15627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:32.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:31 smithi150 ceph-mon[22676]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:32:32.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:31 smithi150 ceph-mon[22676]: from='client.15627 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:32:32.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:32 smithi130 ceph-mon[21867]: Upgrade: Updating node-exporter.a (1/2) 2024-04-22T21:32:32.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:32 smithi130 ceph-mon[21867]: Deploying daemon node-exporter.a on smithi130 2024-04-22T21:32:32.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:32 smithi130 ceph-mon[26394]: Upgrade: Updating node-exporter.a (1/2) 2024-04-22T21:32:32.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:32 smithi130 ceph-mon[26394]: Deploying daemon node-exporter.a on smithi130 2024-04-22T21:32:32.797 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:32 smithi130 systemd[1]: Stopping Ceph node-exporter.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:32:33.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:32 smithi150 ceph-mon[22676]: Upgrade: Updating node-exporter.a (1/2) 2024-04-22T21:32:33.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:32 smithi150 ceph-mon[22676]: Deploying daemon node-exporter.a on smithi130 2024-04-22T21:32:33.228 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:32 smithi130 podman[73316]: 2024-04-22 21:32:32.906361667 +0000 UTC m=+0.141726452 container died 4057e105dc28a04c6a5aea80232a5a6e655b5add4669caa59e395fa1696d74ea (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:32:33.228 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:33 smithi130 podman[73316]: 2024-04-22 21:32:33.245809002 +0000 UTC m=+0.481173804 container cleanup 4057e105dc28a04c6a5aea80232a5a6e655b5add4669caa59e395fa1696d74ea (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:32:33.546 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:33 smithi130 bash[73316]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a 2024-04-22T21:32:33.546 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:33 smithi130 podman[73330]: 2024-04-22 21:32:33.48003916 +0000 UTC m=+0.577951125 container remove 4057e105dc28a04c6a5aea80232a5a6e655b5add4669caa59e395fa1696d74ea (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:32:33.547 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:33 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-22T21:32:33.913 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:33 smithi130 ceph-mon[21867]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 564 B/s rd, 0 op/s 2024-04-22T21:32:33.914 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:33 smithi130 ceph-mon[26394]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 564 B/s rd, 0 op/s 2024-04-22T21:32:33.914 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:33 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-22T21:32:33.914 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:33 smithi130 systemd[1]: Stopped Ceph node-exporter.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:32:33.914 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:33 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 5.252s CPU time. 2024-04-22T21:32:34.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:33 smithi150 ceph-mon[22676]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 564 B/s rd, 0 op/s 2024-04-22T21:32:34.296 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:33 smithi130 systemd[1]: Starting Ceph node-exporter.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:32:34.296 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:34 smithi130 bash[73461]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-22T21:32:34.796 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:34 smithi130 bash[73461]: Getting image source signatures 2024-04-22T21:32:34.796 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:34 smithi130 bash[73461]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-22T21:32:34.796 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:34 smithi130 bash[73461]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-22T21:32:34.797 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:34 smithi130 bash[73461]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-22T21:32:36.046 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:35 smithi130 bash[73461]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-22T21:32:36.047 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:35 smithi130 bash[73461]: Writing manifest to image destination 2024-04-22T21:32:36.487 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-mon[26394]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 711 B/s rd, 0 op/s 2024-04-22T21:32:36.487 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-mon[21867]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 711 B/s rd, 0 op/s 2024-04-22T21:32:36.488 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 podman[73461]: 2024-04-22 21:32:36.163139697 +0000 UTC m=+2.134039289 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-22T21:32:36.488 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 podman[73461]: 2024-04-22 21:32:36.27003209 +0000 UTC m=+2.240931669 container create 660779e0e8355413972dd02c02df7233cfd48d046629a652a97664be53b2d833 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:32:36.488 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 podman[73461]: 2024-04-22 21:32:36.497033943 +0000 UTC m=+2.467933521 container init 660779e0e8355413972dd02c02df7233cfd48d046629a652a97664be53b2d833 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:32:36.488 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 podman[73461]: 2024-04-22 21:32:36.500230489 +0000 UTC m=+2.471130073 container start 660779e0e8355413972dd02c02df7233cfd48d046629a652a97664be53b2d833 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:32:36.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:36 smithi150 ceph-mon[22676]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 711 B/s rd, 0 op/s 2024-04-22T21:32:36.797 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.504Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-22T21:32:36.798 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.504Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-22T21:32:36.798 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.505Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-22T21:32:36.798 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.505Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T21:32:36.798 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.505Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-22T21:32:36.798 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.505Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-22T21:32:36.798 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-22T21:32:36.798 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=arp 2024-04-22T21:32:36.798 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-22T21:32:36.798 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-22T21:32:36.798 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=edac 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-22T21:32:36.799 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-22T21:32:36.800 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-22T21:32:36.800 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-22T21:32:36.800 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-22T21:32:36.800 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-22T21:32:36.800 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-22T21:32:36.800 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-22T21:32:36.800 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-22T21:32:36.800 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-22T21:32:36.800 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=os 2024-04-22T21:32:36.800 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-22T21:32:36.800 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-22T21:32:36.800 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-22T21:32:36.800 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-22T21:32:36.801 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-22T21:32:36.801 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-22T21:32:36.801 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-22T21:32:36.801 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=stat 2024-04-22T21:32:36.801 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-22T21:32:36.801 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-22T21:32:36.801 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-22T21:32:36.801 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=time 2024-04-22T21:32:36.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-22T21:32:36.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=uname 2024-04-22T21:32:36.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-22T21:32:36.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-22T21:32:36.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-22T21:32:36.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-22T21:32:36.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a[73512]: ts=2024-04-22T21:32:36.506Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-22T21:32:36.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 bash[73461]: 660779e0e8355413972dd02c02df7233cfd48d046629a652a97664be53b2d833 2024-04-22T21:32:36.802 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:32:36 smithi130 systemd[1]: Started Ceph node-exporter.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:32:37.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:37 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:37.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:37 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:37.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:37 smithi150 ceph-mon[22676]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:38.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:37 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:38.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:37 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:38.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:37 smithi130 ceph-mon[26394]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:38.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:37 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:38.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:37 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:38.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:37 smithi130 ceph-mon[21867]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:38.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:38 smithi130 ceph-mon[21867]: Upgrade: Updating node-exporter.b (2/2) 2024-04-22T21:32:38.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:38 smithi130 ceph-mon[21867]: Deploying daemon node-exporter.b on smithi150 2024-04-22T21:32:38.899 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:38 smithi130 ceph-mon[26394]: Upgrade: Updating node-exporter.b (2/2) 2024-04-22T21:32:38.899 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:38 smithi130 ceph-mon[26394]: Deploying daemon node-exporter.b on smithi150 2024-04-22T21:32:38.935 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:38 smithi150 ceph-mon[22676]: Upgrade: Updating node-exporter.b (2/2) 2024-04-22T21:32:38.935 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:38 smithi150 ceph-mon[22676]: Deploying daemon node-exporter.b on smithi150 2024-04-22T21:32:39.213 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:38 smithi150 systemd[1]: Stopping Ceph node-exporter.b for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:32:39.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:32:38] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.33.4" 2024-04-22T21:32:39.486 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:39 smithi150 podman[59393]: 2024-04-22 21:32:39.229997329 +0000 UTC m=+0.135087986 container died 2bcf914662579cf3ddad73910129323f0322ef7c2619466ee219b968f05c5931 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:32:39.819 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:39 smithi150 podman[59393]: 2024-04-22 21:32:39.635713777 +0000 UTC m=+0.540804433 container cleanup 2bcf914662579cf3ddad73910129323f0322ef7c2619466ee219b968f05c5931 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:32:39.819 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:39 smithi150 bash[59393]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b 2024-04-22T21:32:40.100 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:39 smithi150 podman[59407]: 2024-04-22 21:32:39.836900722 +0000 UTC m=+0.605660746 container remove 2bcf914662579cf3ddad73910129323f0322ef7c2619466ee219b968f05c5931 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:32:40.100 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:39 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-22T21:32:40.101 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:39 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-22T21:32:40.101 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:39 smithi150 systemd[1]: Stopped Ceph node-exporter.b for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:32:40.101 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:39 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 5.148s CPU time. 2024-04-22T21:32:40.535 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:40 smithi130 ceph-mon[21867]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:40.535 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:40 smithi130 ceph-mon[26394]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:40.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:40 smithi150 ceph-mon[22676]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:40.600 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:40 smithi150 systemd[1]: Starting Ceph node-exporter.b for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:32:40.601 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:40 smithi150 bash[59539]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-22T21:32:41.100 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:40 smithi150 bash[59539]: Getting image source signatures 2024-04-22T21:32:41.100 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:40 smithi150 bash[59539]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-22T21:32:41.100 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:40 smithi150 bash[59539]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-22T21:32:41.100 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:40 smithi150 bash[59539]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-22T21:32:42.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:42 smithi130 ceph-mon[21867]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:32:42.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:42 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:32:42.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:42 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/943588825' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:32:42.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:42 smithi130 ceph-mon[26394]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:32:42.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:42 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:32:42.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:42 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/943588825' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:32:42.600 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:42 smithi150 bash[59539]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-22T21:32:42.600 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:42 smithi150 bash[59539]: Writing manifest to image destination 2024-04-22T21:32:42.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:42 smithi150 ceph-mon[22676]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:32:42.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:42 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:32:42.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:42 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/943588825' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:32:42.977 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:42 smithi150 podman[59539]: 2024-04-22 21:32:42.668677816 +0000 UTC m=+2.394569329 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-22T21:32:42.977 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:42 smithi150 podman[59539]: 2024-04-22 21:32:42.784133827 +0000 UTC m=+2.510025319 container create d5ffcbabfa8ced7ba8c567c2eabc482d8100dc08148f3a0c71f4f85403b34858 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:32:43.231 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:42 smithi150 podman[59539]: 2024-04-22 21:32:42.994692146 +0000 UTC m=+2.720583649 container init d5ffcbabfa8ced7ba8c567c2eabc482d8100dc08148f3a0c71f4f85403b34858 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:32:43.232 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:42 smithi150 podman[59539]: 2024-04-22 21:32:42.997770509 +0000 UTC m=+2.723662001 container start d5ffcbabfa8ced7ba8c567c2eabc482d8100dc08148f3a0c71f4f85403b34858 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:32:43.232 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.001Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-22T21:32:43.232 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.001Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-22T21:32:43.232 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.002Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-22T21:32:43.232 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.002Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-22T21:32:43.232 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.002Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-22T21:32:43.232 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T21:32:43.232 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-22T21:32:43.232 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=arp 2024-04-22T21:32:43.232 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-22T21:32:43.232 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-22T21:32:43.233 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-22T21:32:43.233 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-22T21:32:43.233 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-22T21:32:43.233 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-22T21:32:43.233 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-22T21:32:43.233 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-22T21:32:43.233 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=edac 2024-04-22T21:32:43.233 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-22T21:32:43.233 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-22T21:32:43.234 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=os 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=stat 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=time 2024-04-22T21:32:43.235 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-22T21:32:43.236 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=uname 2024-04-22T21:32:43.236 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-22T21:32:43.236 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-22T21:32:43.236 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-22T21:32:43.236 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-22T21:32:43.236 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b[59590]: ts=2024-04-22T21:32:43.003Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-22T21:32:43.236 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 bash[59539]: d5ffcbabfa8ced7ba8c567c2eabc482d8100dc08148f3a0c71f4f85403b34858 2024-04-22T21:32:43.236 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:32:43 smithi150 systemd[1]: Started Ceph node-exporter.b for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:32:44.209 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:44 smithi150 ceph-mon[22676]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:32:44.209 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:44 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:44.209 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:44 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:44.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:44 smithi130 ceph-mon[21867]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:32:44.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:44 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:44.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:44 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:44.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:44 smithi130 ceph-mon[26394]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:32:44.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:44 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:44.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:44 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:46.539 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:46 smithi150 ceph-mon[22676]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:32:46.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:46 smithi130 ceph-mon[21867]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:32:46.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:46 smithi130 ceph-mon[26394]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:32:48.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:48 smithi130 ceph-mon[21867]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:32:48.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:48 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:48.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:48 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:48.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:48 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:48.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:48 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:48.900 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:48 smithi130 ceph-mon[26394]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:32:48.900 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:48 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:48.900 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:48 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:48.900 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:48 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:48.900 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:48 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:49.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:48 smithi150 ceph-mon[22676]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:32:49.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:48 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:49.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:48 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:49.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:48 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:49.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:48 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:49.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:32:48] "GET /metrics HTTP/1.1" 200 37919 "" "Prometheus/2.33.4" 2024-04-22T21:32:50.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:49 smithi130 ceph-mon[21867]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:49 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:49 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:49 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:49 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:50.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:49 smithi130 ceph-mon[26394]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:50.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:49 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:50.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:49 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:50.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:49 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:50.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:49 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:50.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:49 smithi150 ceph-mon[22676]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:50.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:49 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:50.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:49 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:50.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:49 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:50.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:49 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:51.802 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:51 smithi150 ceph-mon[22676]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:51.802 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:51 smithi150 ceph-mon[22676]: from='client.25525 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:32:52.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:51 smithi130 ceph-mon[26394]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:52.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:51 smithi130 ceph-mon[26394]: from='client.25525 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:32:52.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:51 smithi130 ceph-mon[21867]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:52.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:51 smithi130 ceph-mon[21867]: from='client.25525 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:32:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:32:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:32:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:32:53.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:53.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:32:53.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:32:53.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:53.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:53.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:53.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:53 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:32:53.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:53 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:53 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:53 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:53 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:53 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:53.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:53 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:32:53.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:53 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:32:53.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:53 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:32:53.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:53 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:53.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:53 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:32:53.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:53 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:32:53.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:53 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:32:54.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:54 smithi130 ceph-mon[21867]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:54.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:54 smithi130 ceph-mon[26394]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:54.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:54 smithi150 ceph-mon[22676]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:32:55.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:32:55.387+0000 7fdddea21700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:55.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:32:55.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:55.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:55.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:55.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:55.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:55.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:32:55.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:55.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:32:55.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:55.797 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:32:55.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:55.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:55.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:55.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:55.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:55.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:55.798 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:55.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:55.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:55.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:32:55.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:32:55.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:32:55.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:32:55.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:32:55.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:32:55.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:32:55.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:32:55.799 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return future.result(timeout) 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.__get_result() 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: raise self._exception 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: use_current_daemon_image=reconfig, 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:55.800 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:55.801 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:56.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[21867]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T21:32:56.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:56.547 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:56.547 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:56.547 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:56.547 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:56.547 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:56.547 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:56.547 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:56.548 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: return future.result(timeout) 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.__get_result() 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: raise self._exception 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:32:56.549 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:56.550 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:32:56.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:56.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:56.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:56.551 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:56.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[26394]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T21:32:56.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[26394]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:56.551 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:56.551 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:56.551 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:56.551 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:56.551 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:56.552 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: return future.result(timeout) 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.__get_result() 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: raise self._exception 2024-04-22T21:32:56.553 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:56.554 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:56.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:32:56.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:56.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:56.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:56.555 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:56.555 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:56.555 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:56.555 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:56.555 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:56.555 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:56.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:32:56.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:56.556 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:56.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:56.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:56.557 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:56.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:32:56.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:56 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:56 smithi150 ceph-mon[22676]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T21:32:56.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:56 smithi150 ceph-mon[22676]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:56.600 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:56.600 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:32:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:32:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:56.601 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:32:56.602 INFO:journalctl@ceph.mon.b.smithi150.stdout: return future.result(timeout) 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.__get_result() 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: raise self._exception 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:32:56.603 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:32:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:32:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:32:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:32:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:32:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:32:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:32:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:32:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:32:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:32:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:32:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:32:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:32:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:32:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:56 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:32:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:56 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:56 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:56 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:56 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:56 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:56 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:56 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:56 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:56 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:32:57.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:32:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:32:57.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:32:58.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:58 smithi130 ceph-mon[26394]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:32:58.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:58 smithi130 ceph-mon[26394]: Upgrade: Updating prometheus.a 2024-04-22T21:32:58.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:32:58 smithi130 ceph-mon[26394]: Deploying daemon prometheus.a on smithi150 2024-04-22T21:32:58.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:58 smithi130 ceph-mon[21867]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:32:58.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:58 smithi130 ceph-mon[21867]: Upgrade: Updating prometheus.a 2024-04-22T21:32:58.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:32:58 smithi130 ceph-mon[21867]: Deploying daemon prometheus.a on smithi150 2024-04-22T21:32:58.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:58 smithi150 ceph-mon[22676]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:32:58.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:58 smithi150 ceph-mon[22676]: Upgrade: Updating prometheus.a 2024-04-22T21:32:58.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:32:58 smithi150 ceph-mon[22676]: Deploying daemon prometheus.a on smithi150 2024-04-22T21:32:59.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:32:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:32:58] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.33.4" 2024-04-22T21:33:00.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:00 smithi130 ceph-mon[21867]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T21:33:00.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:00 smithi130 ceph-mon[26394]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T21:33:00.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:00 smithi150 ceph-mon[22676]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T21:33:01.337 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:33:01.994 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:33:02.329 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (13m) 14s ago 21m 24.0M - 0.23.0 ba2b418f427c 23fd04f877ce 2024-04-22T21:33:02.329 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (13m) 14s ago 21m 50.9M - 8.3.5 dad864ee21e9 e021384d6729 2024-04-22T21:33:02.329 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (21s) 14s ago 20m 48.0M - 3.5 e1d6a67b021e 748791bac752 2024-04-22T21:33:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (44s) 14s ago 26m 451M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:33:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (10m) 14s ago 28m 501M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:33:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (28m) 14s ago 28m 79.7M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:33:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (26m) 14s ago 26m 55.2M 2048M 17.2.0 e1d6a67b021e 861909a68cf9 2024-04-22T21:33:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (26m) 14s ago 26m 56.8M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:33:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (25s) 14s ago 22m 3393k - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:33:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (19s) 14s ago 22m 3644k - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:33:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (26m) 14s ago 26m 54.9M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:33:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (25m) 14s ago 25m 57.2M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:33:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (25m) 14s ago 25m 50.5M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:33:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (24m) 14s ago 24m 53.0M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:33:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (24m) 14s ago 24m 55.4M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:33:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (24m) 14s ago 24m 52.6M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:33:02.331 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (23m) 14s ago 23m 54.3M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:33:02.331 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (23m) 14s ago 23m 53.7M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:33:02.331 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (50s) 14s ago 22m 60.1M - 2.33.4 514e6a882f6e bcd947b38040 2024-04-22T21:33:02.331 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (20m) 14s ago 20m 77.8M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:33:02.331 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (20m) 14s ago 20m 75.0M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:33:02.403 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:33:02.403 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:33:02.403 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T21:33:02.403 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": {}, 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:33:02.404 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:33:02.405 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:33:02.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:02 smithi130 ceph-mon[21867]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:02.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:02 smithi130 ceph-mon[21867]: from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:02.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:02 smithi130 ceph-mon[26394]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:02.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:02 smithi130 ceph-mon[26394]: from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:02.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:02 smithi150 ceph-mon[22676]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:02.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:02 smithi150 ceph-mon[22676]: from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:02.739 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:33:02.739 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:33:02.739 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:33:02.740 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-22T21:33:02.740 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-04-22T21:33:02.740 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-04-22T21:33:02.740 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-04-22T21:33:02.740 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "2/2 daemons upgraded", 2024-04-22T21:33:02.740 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading prometheus daemons", 2024-04-22T21:33:02.740 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:33:02.740 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:33:03.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:03 smithi130 ceph-mon[26394]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:03.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:03 smithi130 ceph-mon[26394]: from='client.25552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:03.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:03 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2782439819' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:03.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:03 smithi130 ceph-mon[21867]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:03.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:03 smithi130 ceph-mon[21867]: from='client.25552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:03.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:03 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2782439819' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:03.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:03 smithi150 ceph-mon[22676]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:03.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:03 smithi150 ceph-mon[22676]: from='client.25552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:03.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:03 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2782439819' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:04.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:04 smithi150 ceph-mon[22676]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:04.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:04 smithi150 ceph-mon[22676]: from='client.25564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:04.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:04 smithi130 ceph-mon[26394]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:04.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:04 smithi130 ceph-mon[26394]: from='client.25564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:04.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:04 smithi130 ceph-mon[21867]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:04.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:04 smithi130 ceph-mon[21867]: from='client.25564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:06.479 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:06 smithi130 ceph-mon[26394]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:06.480 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:06 smithi130 ceph-mon[21867]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:06.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:06 smithi150 ceph-mon[22676]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:07.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:07 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2867080484' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:33:07.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:07 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2867080484' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:33:07.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:07 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2867080484' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:33:08.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:08 smithi150 ceph-mon[22676]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:08.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:08 smithi130 ceph-mon[21867]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:08.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:08 smithi130 ceph-mon[26394]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:09.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:33:08] "GET /metrics HTTP/1.1" 200 37920 "" "Prometheus/2.33.4" 2024-04-22T21:33:10.029 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:09 smithi150 systemd[1]: Stopping Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:33:10.305 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:33:10.046Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T21:33:10.305 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:33:10.047Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T21:33:10.306 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:33:10.047Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T21:33:10.306 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:33:10.047Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T21:33:10.306 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:33:10.047Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T21:33:10.306 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:33:10.047Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T21:33:10.306 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:33:10.047Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T21:33:10.306 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:33:10.047Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T21:33:10.306 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:33:10.047Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T21:33:10.306 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:33:10.047Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T21:33:10.306 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:33:10.047Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T21:33:10.306 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[57508]: ts=2024-04-22T21:33:10.047Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T21:33:10.306 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 podman[61086]: 2024-04-22 21:33:10.05702298 +0000 UTC m=+0.200502750 container died bcd947b380404de8071f92c34445d2f728dc4fd888ad0a84b9e181918e299bbd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:33:10.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:10 smithi150 ceph-mon[22676]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:33:10.600 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 podman[61086]: 2024-04-22 21:33:10.429659246 +0000 UTC m=+0.573139009 container cleanup bcd947b380404de8071f92c34445d2f728dc4fd888ad0a84b9e181918e299bbd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:33:10.601 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 bash[61086]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T21:33:10.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:10 smithi130 ceph-mon[21867]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:33:10.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:10 smithi130 ceph-mon[26394]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T21:33:11.018 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 podman[61100]: 2024-04-22 21:33:10.680772572 +0000 UTC m=+0.626040986 container remove bcd947b380404de8071f92c34445d2f728dc4fd888ad0a84b9e181918e299bbd (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:33:11.018 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T21:33:11.018 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 systemd[1]: Stopped Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:33:11.018 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:10 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.293s CPU time. 2024-04-22T21:33:11.350 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 systemd[1]: Starting Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:33:11.350 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 podman[61230]: 2024-04-22 21:33:11.154798672 +0000 UTC m=+0.019398276 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-22T21:33:11.350 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 podman[61230]: 2024-04-22 21:33:11.274294338 +0000 UTC m=+0.138893928 container create a8e5ee53ede38be89c93cd3198b62c6b1a0510fe4ee43ca432dbe4279cf3825e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:33:11.850 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 podman[61230]: 2024-04-22 21:33:11.518049842 +0000 UTC m=+0.382649433 container init a8e5ee53ede38be89c93cd3198b62c6b1a0510fe4ee43ca432dbe4279cf3825e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 podman[61230]: 2024-04-22 21:33:11.520670068 +0000 UTC m=+0.385269669 container start a8e5ee53ede38be89c93cd3198b62c6b1a0510fe4ee43ca432dbe4279cf3825e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.544Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.544Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.544Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi150 (none))" 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.544Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.544Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.546Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.547Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.547Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.547Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.550Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.550Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.528µs 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.550Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T21:33:11.851 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.556Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.570Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.580Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.582Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.584Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.586Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.586Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.586Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=87.626µs wal_replay_duration=35.78491ms wbl_replay_duration=120ns total_replay_duration=35.918846ms 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.589Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.589Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.589Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.610Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.30982ms db_storage=937ns remote_storage=1.179µs web_handler=795ns query_engine=586ns scrape=375.841µs scrape_sd=103.316µs notify=12.723µs notify_sd=17.459µs rules=20.478667ms tracing=6.028µs 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.611Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:33:11.611Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-22T21:33:11.852 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 bash[61230]: a8e5ee53ede38be89c93cd3198b62c6b1a0510fe4ee43ca432dbe4279cf3825e 2024-04-22T21:33:11.853 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:33:11 smithi150 systemd[1]: Started Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:33:12.575 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:12 smithi150 ceph-mon[22676]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:12.575 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:12 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:12.575 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:12 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:12.575 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:12 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:33:12.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:12 smithi130 ceph-mon[21867]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:12.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:12 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:12 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:12 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:33:12.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:12 smithi130 ceph-mon[26394]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:12.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:12 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:12.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:12 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:12.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:12 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:33:14.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:14 smithi150 ceph-mon[22676]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:14.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:14 smithi130 ceph-mon[21867]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:14.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:14 smithi130 ceph-mon[26394]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:16.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:16 smithi150 ceph-mon[22676]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:16.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:16 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:16.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:16 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:16.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:16 smithi130 ceph-mon[21867]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:16.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:16 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:16.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:16 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:16.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:16 smithi130 ceph-mon[26394]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:16.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:16 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:16.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:16 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:18.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:18 smithi150 ceph-mon[22676]: from='client.25579 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:33:18.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:18 smithi150 ceph-mon[22676]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:18.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:18 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:18.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:18 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:18.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:18 smithi130 ceph-mon[21867]: from='client.25579 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:33:18.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:18 smithi130 ceph-mon[21867]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:18.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:18.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:18.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:18 smithi130 ceph-mon[26394]: from='client.25579 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:33:18.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:18 smithi130 ceph-mon[26394]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:18.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:18.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:19.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:33:18] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.43.0" 2024-04-22T21:33:20.484 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:20 smithi150 ceph-mon[22676]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T21:33:20.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:20 smithi130 ceph-mon[26394]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T21:33:20.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:20 smithi130 ceph-mon[21867]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T21:33:21.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:21.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:21.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:33:21.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:33:21.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:21.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:33:21.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:33:21.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:33:21.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:21.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:21.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:33:21.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:33:21.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:21.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:33:21.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:33:21.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:21 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:33:21.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:21 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:21.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:21 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:21.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:21 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:33:21.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:21 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:33:21.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:21 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:21.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:21 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:33:21.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:21 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:33:21.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:21 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:33:22.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:22 smithi150 ceph-mon[22676]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:33:22.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:22 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:33:22.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:22 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:33:22.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:22 smithi130 ceph-mon[26394]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:33:22.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:22 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:33:22.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:22 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:33:22.883 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:22 smithi130 ceph-mon[21867]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:33:22.883 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:22 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:33:22.883 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:22 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:33:24.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:33:24.140+0000 7fdddea21700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:24.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:33:24.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:24.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:24.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:24.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:24.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:24.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:33:24.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:24.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:33:24.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:24.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:33:24.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:24.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:24.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:24.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:24.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:24.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:24.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:24.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:24.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:24.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:33:24.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:33:24.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:33:24.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:33:24.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:33:24.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:33:24.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:33:24.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return future.result(timeout) 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.__get_result() 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: raise self._exception 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: use_current_daemon_image=reconfig, 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:24.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:24.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:24.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:24.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:24 smithi150 ceph-mon[22676]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:33:24.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:24 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:33:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:24 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:33:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:24 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:24 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:24 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:24 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:24 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:24 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:24 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:24 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:24 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:24 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[26394]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:33:24.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:33:24.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:33:24.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.904 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[21867]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:33:24.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:33:24.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:33:24.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:24.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:24 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:25.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:25 smithi150 ceph-mon[22676]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:25.850 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:33:25.850 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:33:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:33:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:33:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:25.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:33:25.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: return future.result(timeout) 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.__get_result() 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: raise self._exception 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:25.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:25.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:25.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:25.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:33:25.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:25.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:33:25.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:25.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:33:25.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:25.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:25.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:25.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:25.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:25.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:25.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:25.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:25.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:25.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:25 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:33:26.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:25 smithi130 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:33:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:33:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:33:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:33:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:26.047 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:33:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:33:26.048 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: return future.result(timeout) 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.__get_result() 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: raise self._exception 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:33:26.049 INFO:journalctl@ceph.mon.a.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:26.050 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:33:26.050 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:26.050 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:26.050 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:26.050 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:26.050 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:26.050 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:33:26.050 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:26.050 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:33:26.050 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:26.050 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:33:26.050 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:26.050 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:26.051 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:26.051 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:26.051 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:26.051 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:26.051 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:26.051 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:26.051 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:26.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:25 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:33:26.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:25 smithi130 ceph-mon[26394]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:26.052 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:33:26.052 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:26.052 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:26.052 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:26.052 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:26.052 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:26.052 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:33:26.052 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:26.052 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:33:26.052 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:26.052 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:33:26.053 INFO:journalctl@ceph.mon.c.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: return future.result(timeout) 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.__get_result() 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: raise self._exception 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:26.054 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:33:26.055 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:26.055 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:26.055 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:26.055 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:26.055 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:26.055 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:33:26.055 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:26.055 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:33:26.055 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:26.055 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:33:26.055 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:26.056 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:26.056 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:26.056 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:26.056 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:26.056 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:26.056 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:26.056 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:26.056 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:26.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:25 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T21:33:26.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:26 smithi150 ceph-mon[22676]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:33:27.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:26 smithi130 ceph-mon[26394]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:33:27.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:26 smithi130 ceph-mon[21867]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:33:27.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:27 smithi150 ceph-mon[22676]: Upgrade: Updating alertmanager.a 2024-04-22T21:33:27.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:27 smithi150 ceph-mon[22676]: Deploying daemon alertmanager.a on smithi130 2024-04-22T21:33:27.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:27 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:33:28.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:27 smithi130 ceph-mon[21867]: Upgrade: Updating alertmanager.a 2024-04-22T21:33:28.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:27 smithi130 ceph-mon[21867]: Deploying daemon alertmanager.a on smithi130 2024-04-22T21:33:28.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:27 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:33:28.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:27 smithi130 ceph-mon[26394]: Upgrade: Updating alertmanager.a 2024-04-22T21:33:28.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:27 smithi130 ceph-mon[26394]: Deploying daemon alertmanager.a on smithi130 2024-04-22T21:33:28.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:27 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:33:28.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:28 smithi150 ceph-mon[22676]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:33:28.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:28 smithi130 ceph-mon[21867]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:33:28.900 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:28 smithi130 ceph-mon[26394]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:33:29.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:33:28] "GET /metrics HTTP/1.1" 200 37922 "" "Prometheus/2.43.0" 2024-04-22T21:33:30.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:30 smithi130 ceph-mon[26394]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:33:30.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:30 smithi130 ceph-mon[21867]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:33:30.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:30 smithi150 ceph-mon[22676]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:33:32.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:32 smithi150 ceph-mon[22676]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:33:33.015 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:32 smithi130 ceph-mon[21867]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:33:33.018 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:32 smithi130 ceph-mon[26394]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:33:33.100 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:33:33.679 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:33 smithi130 systemd[1]: Stopping Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:33:33.817 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:33:33.818 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (14m) 46s ago 22m 24.0M - 0.23.0 ba2b418f427c 23fd04f877ce 2024-04-22T21:33:33.818 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (14m) 17s ago 21m 50.9M - 8.3.5 dad864ee21e9 e021384d6729 2024-04-22T21:33:33.818 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (52s) 46s ago 21m 48.0M - 3.5 e1d6a67b021e 748791bac752 2024-04-22T21:33:33.818 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (76s) 17s ago 26m 451M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:33:33.818 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (10m) 46s ago 28m 501M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:33:33.818 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (28m) 46s ago 28m 79.7M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:33:33.818 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (27m) 17s ago 27m 51.6M 2048M 17.2.0 e1d6a67b021e 861909a68cf9 2024-04-22T21:33:33.819 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (27m) 46s ago 27m 56.8M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:33:33.819 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (57s) 46s ago 22m 3393k - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:33:33.819 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (51s) 17s ago 22m 8258k - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:33:33.819 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (26m) 46s ago 26m 54.9M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:33:33.819 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (26m) 46s ago 26m 57.2M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:33:33.819 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (25m) 46s ago 25m 50.5M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:33:33.819 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (25m) 46s ago 25m 53.0M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:33:33.819 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (25m) 17s ago 25m 55.3M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:33:33.819 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (24m) 17s ago 24m 52.6M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:33:33.819 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (24m) 17s ago 24m 54.4M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:33:33.820 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (23m) 17s ago 23m 53.7M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:33:33.820 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (22s) 17s ago 23m 46.0M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:33:33.820 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (21m) 46s ago 21m 77.8M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:33:33.820 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (21m) 17s ago 21m 75.1M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:33:34.014 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[55940]: level=info ts=2024-04-22T21:33:33.696Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-22T21:33:34.015 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:33 smithi130 podman[76019]: 2024-04-22 21:33:33.737292942 +0000 UTC m=+0.169951549 container stop 23fd04f877ce9baea4704909657f4398261f969593a5035cea5f8ef2fcd63505 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:33:34.015 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:33 smithi130 podman[76019]: 2024-04-22 21:33:33.762802685 +0000 UTC m=+0.195461290 container died 23fd04f877ce9baea4704909657f4398261f969593a5035cea5f8ef2fcd63505 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:33:34.221 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:33:34.221 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:33:34.221 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T21:33:34.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:33:34.221 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:33:34.221 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:33:34.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:33:34.222 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:33:34.222 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:33:34.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:33:34.222 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": {}, 2024-04-22T21:33:34.222 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:33:34.222 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:33:34.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:33:34.222 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:33:34.222 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T21:33:34.222 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:33:34.222 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:33:34.222 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:33:34.296 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:34 smithi130 podman[76019]: 2024-04-22 21:33:34.24784316 +0000 UTC m=+0.680501767 container cleanup 23fd04f877ce9baea4704909657f4398261f969593a5035cea5f8ef2fcd63505 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:33:34.297 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:34 smithi130 bash[76019]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a 2024-04-22T21:33:34.576 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:33:34.577 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:33:34.577 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:33:34.577 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-22T21:33:34.577 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-04-22T21:33:34.577 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-04-22T21:33:34.577 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-04-22T21:33:34.577 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "2/2 daemons upgraded", 2024-04-22T21:33:34.577 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading alertmanager daemons", 2024-04-22T21:33:34.577 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:33:34.577 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:33:34.765 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:34 smithi130 podman[76053]: 2024-04-22 21:33:34.466299593 +0000 UTC m=+0.723787170 container remove 23fd04f877ce9baea4704909657f4398261f969593a5035cea5f8ef2fcd63505 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:33:34.765 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:34 smithi130 podman[76053]: 2024-04-22 21:33:34.550320014 +0000 UTC m=+0.807807593 volume remove fe3bb1df2711f659ed6c559592f2bcdc3aafd6bda65efe37147783786bd43e56 2024-04-22T21:33:34.766 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:34 smithi130 ceph-mon[26394]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:33:34.766 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:34 smithi130 ceph-mon[26394]: from='client.25594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:34.766 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:34 smithi130 ceph-mon[26394]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:34.766 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:34 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2605141705' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:34.766 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:34 smithi130 ceph-mon[21867]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:33:34.766 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:34 smithi130 ceph-mon[21867]: from='client.25594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:34.766 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:34 smithi130 ceph-mon[21867]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:34.766 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:34 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2605141705' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:34.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:34 smithi150 ceph-mon[22676]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:33:34.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:34 smithi150 ceph-mon[22676]: from='client.25594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:34.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:34 smithi150 ceph-mon[22676]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:34.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:34 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2605141705' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:35.600 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:35 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-22T21:33:35.600 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:35 smithi130 systemd[1]: Stopped Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:33:35.600 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:35 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.531s CPU time. 2024-04-22T21:33:35.933 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:35 smithi130 ceph-mon[21867]: from='client.15723 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:35.933 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:35 smithi130 ceph-mon[21867]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:35.933 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:35 smithi130 ceph-mon[26394]: from='client.15723 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:35.934 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:35 smithi130 ceph-mon[26394]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:35.934 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:35 smithi130 systemd[1]: Starting Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:33:36.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:35 smithi150 ceph-mon[22676]: from='client.15723 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:36.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:35 smithi150 ceph-mon[22676]: from='client.15735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:33:36.182 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:35 smithi130 podman[76376]: 2024-04-22 21:33:35.854071542 +0000 UTC m=+0.022803076 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-22T21:33:36.182 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:35 smithi130 podman[76376]: 2024-04-22 21:33:35.950093144 +0000 UTC m=+0.118824690 volume create cc9e6614baf79bae72709d72911f42c6120ac9f2b632e32b186cf68b8413bf2c 2024-04-22T21:33:36.182 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:36 smithi130 podman[76376]: 2024-04-22 21:33:36.08408925 +0000 UTC m=+0.252820783 container create 49cc4394f696e3d77ce5d78fc7595ef45a90984cca68a514642d5c9bc55298a0 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:33:36.507 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:36 smithi130 podman[76376]: 2024-04-22 21:33:36.344847021 +0000 UTC m=+0.513578552 container init 49cc4394f696e3d77ce5d78fc7595ef45a90984cca68a514642d5c9bc55298a0 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:33:36.507 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:36 smithi130 podman[76376]: 2024-04-22 21:33:36.347416568 +0000 UTC m=+0.516148109 container start 49cc4394f696e3d77ce5d78fc7595ef45a90984cca68a514642d5c9bc55298a0 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:33:36.507 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:33:36.358Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-22T21:33:36.507 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:33:36.358Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-22T21:33:36.507 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:33:36.360Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.130 port=9094 2024-04-22T21:33:36.507 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:33:36.361Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-22T21:33:36.507 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:33:36.387Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T21:33:36.507 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:33:36.387Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T21:33:36.507 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:33:36.390Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-22T21:33:36.507 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:33:36.390Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-22T21:33:36.508 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:36 smithi130 bash[76376]: 49cc4394f696e3d77ce5d78fc7595ef45a90984cca68a514642d5c9bc55298a0 2024-04-22T21:33:36.508 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:36 smithi130 systemd[1]: Started Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:33:36.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-mon[21867]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:33:36.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2016782789' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:33:36.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:36.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:36.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-mon[26394]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:33:36.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2016782789' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:33:36.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:36.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:36 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:37.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:36 smithi150 ceph-mon[22676]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-22T21:33:37.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:36 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2016782789' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:33:37.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:36 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:37.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:36 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:38.605 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:33:38.362Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000299771s 2024-04-22T21:33:38.898 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:38 smithi130 ceph-mon[21867]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:33:38.898 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:38 smithi130 ceph-mon[26394]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:33:38.926 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:38 smithi150 ceph-mon[22676]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:33:39.231 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:33:38] "GET /metrics HTTP/1.1" 200 37922 "" "Prometheus/2.43.0" 2024-04-22T21:33:40.652 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:40 smithi150 ceph-mon[22676]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:40.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:40 smithi130 ceph-mon[21867]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:40.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:40 smithi130 ceph-mon[26394]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:42.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:41 smithi130 ceph-mon[26394]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:42.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:41 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:42.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:41 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:42.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:41 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:42.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:41 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:42.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:41 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:33:42.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:41 smithi130 ceph-mon[21867]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:42.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:41 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:42.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:41 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:42.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:41 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:42.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:41 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:42.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:41 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:33:42.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:41 smithi150 ceph-mon[22676]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:42.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:41 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:42.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:41 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:42.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:41 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:42.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:41 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:42.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:41 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:33:43.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:43 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:43.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:43 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:43.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:43 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:43.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:43 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:43.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:43 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:43.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:43 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:44.185 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:44 smithi130 ceph-mon[21867]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:44.186 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:44 smithi130 ceph-mon[26394]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:44.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:44 smithi150 ceph-mon[22676]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:33:46.569 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:33:46.364Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002770878s 2024-04-22T21:33:46.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:46 smithi150 ceph-mon[22676]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:46.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:46 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:46.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:46 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:46.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:46 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:33:46.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:46 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:33:46.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:46 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:46.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:46 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:33:46.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:46 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:33:46.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:46 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:33:47.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[21867]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:47.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:47.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:47.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:33:47.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:33:47.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:47.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:33:47.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:33:47.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:33:47.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[26394]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:47.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:47.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:47.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:33:47.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:33:47.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:47.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:33:47.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:33:47.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:46 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:33:47.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:47 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:33:47.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:47 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:33:47.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:47 smithi130 ceph-mon[26394]: from='client.25636 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:33:47.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:47 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:33:47.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:47 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:33:47.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:47 smithi130 ceph-mon[21867]: from='client.25636 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:33:47.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:47 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:33:47.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:47 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:33:47.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:47 smithi150 ceph-mon[22676]: from='client.25636 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:33:48.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-mon[26394]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:48.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-mon[21867]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:48.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:48 smithi150 ceph-mon[22676]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:33:49.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:33:48] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.43.0" 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:33:48.973+0000 7fdddea21700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:49.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:49.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:49.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:49.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:49.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:49.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:49.298 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:49.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:49.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:33:49.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:33:49.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:33:49.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:33:49.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:33:49.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:33:49.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:33:49.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:33:49.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return future.result(timeout) 2024-04-22T21:33:49.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:33:49.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.__get_result() 2024-04-22T21:33:49.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: raise self._exception 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: use_current_daemon_image=reconfig, 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:33:49.300 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:33:48] ENGINE Bus STOPPING 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:33:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T21:33:49.301 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:33:49] ENGINE Bus STOPPED 2024-04-22T21:33:49.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:33:49] ENGINE Bus STARTING 2024-04-22T21:33:49.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:33:49] ENGINE Serving on http://:::9283 2024-04-22T21:33:49.302 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:33:49] ENGINE Bus STARTED 2024-04-22T21:33:49.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:33:49.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130:9093"}]: dispatch 2024-04-22T21:33:49.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:49.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:33:49.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:49.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:49.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:49.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:49.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:49.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:49.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:49.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:49.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:49.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:49.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:49 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:33:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130:9093"}]: dispatch 2024-04-22T21:33:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:33:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:33:50.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130:9093"}]: dispatch 2024-04-22T21:33:50.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:33:50.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:33:50.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:33:50.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:50 smithi150 ceph-mon[22676]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T21:33:50.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:50 smithi150 ceph-mon[22676]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:33:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:33:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:33:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:33:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:50.851 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:33:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:33:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:33:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:33:50.852 INFO:journalctl@ceph.mon.b.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:33:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:33:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:33:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:33:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: return future.result(timeout) 2024-04-22T21:33:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:33:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.__get_result() 2024-04-22T21:33:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:33:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: raise self._exception 2024-04-22T21:33:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:33:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:33:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:33:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:33:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:50.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:33:50.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:50.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:50.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:50.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:50.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:50.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:33:50.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:50.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:33:50.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:50.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:33:50.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:50.854 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:50.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:50.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:50.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:50.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:50.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:50.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:50.855 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:50.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:50 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:33:50.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:50 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130:9093"}]: dispatch 2024-04-22T21:33:51.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:50 smithi130 ceph-mon[26394]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T21:33:51.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:50 smithi130 ceph-mon[26394]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:51.047 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:33:51.047 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:51.047 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:51.047 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:51.047 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:51.047 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:51.047 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:33:51.047 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:51.047 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:33:51.047 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:51.047 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:33:51.047 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:51.048 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:51.048 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:51.048 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:51.048 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:51.048 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:51.048 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:51.048 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:51.048 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:51.048 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:33:51.048 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:33:51.048 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:33:51.048 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:33:51.049 INFO:journalctl@ceph.mon.c.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:33:51.049 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:33:51.049 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:33:51.049 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:33:51.049 INFO:journalctl@ceph.mon.c.smithi130.stdout: return future.result(timeout) 2024-04-22T21:33:51.049 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:33:51.049 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.__get_result() 2024-04-22T21:33:51.049 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:33:51.049 INFO:journalctl@ceph.mon.c.smithi130.stdout: raise self._exception 2024-04-22T21:33:51.049 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:33:51.049 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:33:51.049 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:33:51.049 INFO:journalctl@ceph.mon.c.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:33:51.050 INFO:journalctl@ceph.mon.c.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:51.050 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:33:51.050 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:51.050 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:51.050 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:51.050 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:51.050 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:51.050 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:33:51.050 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:51.050 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:33:51.050 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:51.051 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:33:51.051 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:51.051 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:51.051 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:51.051 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:51.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:50 smithi130 ceph-mon[21867]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T21:33:51.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:50 smithi130 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:51.051 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:33:51.051 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:51.051 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:51.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:51.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:51.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:51.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:33:51.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:51.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:33:51.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:51.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:33:51.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:51.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:51.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:51.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:51.052 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:51.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:51.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:51.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:51.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:51.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:33:51.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:33:51.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:33:51.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:33:51.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:33:51.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:33:51.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:33:51.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:33:51.053 INFO:journalctl@ceph.mon.a.smithi130.stdout: return future.result(timeout) 2024-04-22T21:33:51.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:33:51.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.__get_result() 2024-04-22T21:33:51.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:33:51.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: raise self._exception 2024-04-22T21:33:51.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:33:51.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:33:51.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:33:51.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:33:51.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:33:51.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:33:51.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:33:51.054 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:33:51.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:33:51.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:51.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:51.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:33:51.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:33:51.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:33:51.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:33:51.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:33:51.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:33:51.055 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:33:51.056 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:33:51.056 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:33:51.056 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:51.056 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:51.056 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:51.056 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:51.056 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:51.056 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:50 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:33:51.056 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:50 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130:9093"}]: dispatch 2024-04-22T21:33:51.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:33:51.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:33:51.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:33:51.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:33:51.057 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:33:51.057 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:50 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T21:33:51.057 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:50 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130:9093"}]: dispatch 2024-04-22T21:33:52.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:52 smithi130 ceph-mon[21867]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:33:52.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:52 smithi130 ceph-mon[21867]: Upgrade: Updating grafana.a 2024-04-22T21:33:52.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:52 smithi130 ceph-mon[21867]: Deploying daemon grafana.a on smithi150 2024-04-22T21:33:52.849 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:52 smithi130 ceph-mon[26394]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:33:52.849 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:52 smithi130 ceph-mon[26394]: Upgrade: Updating grafana.a 2024-04-22T21:33:52.849 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:52 smithi130 ceph-mon[26394]: Deploying daemon grafana.a on smithi150 2024-04-22T21:33:52.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:52 smithi150 ceph-mon[22676]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:33:52.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:52 smithi150 ceph-mon[22676]: Upgrade: Updating grafana.a 2024-04-22T21:33:52.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:52 smithi150 ceph-mon[22676]: Deploying daemon grafana.a on smithi150 2024-04-22T21:33:54.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:54 smithi150 ceph-mon[22676]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:33:54.996 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:54 smithi130 ceph-mon[21867]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:33:54.996 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:54 smithi130 ceph-mon[26394]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:33:57.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:56 smithi130 ceph-mon[21867]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:33:57.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:56 smithi130 ceph-mon[26394]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:33:57.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:56 smithi150 ceph-mon[22676]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:33:58.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:33:58.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:33:58.058 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:33:58.745 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:33:58 smithi150 ceph-mon[22676]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:33:58.900 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:33:58 smithi130 ceph-mon[21867]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:33:58.900 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:33:58 smithi130 ceph-mon[26394]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:33:59.288 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:33:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:33:58] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.43.0" 2024-04-22T21:34:00.742 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:00 smithi150 systemd[1]: Stopping Ceph grafana.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:34:00.743 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:00 smithi150 ceph-mon[22676]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:34:00.743 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:00 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/588219990' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:34:01.035 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:00 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[46572]: t=2024-04-22T21:34:00+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-22T21:34:01.035 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:00 smithi150 podman[63321]: 2024-04-22 21:34:00.769829916 +0000 UTC m=+0.128881927 container died e021384d6729b2adc301b98aa19981c5ae7b00e7cb411116ec5c20c28db490d5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, vcs-type=git, vendor=Red Hat, Inc., release=236.1648460182, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.openshift.expose-services=, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-22T21:34:01.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:00 smithi130 ceph-mon[21867]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:34:01.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:00 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/588219990' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:34:01.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:00 smithi130 ceph-mon[26394]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:34:01.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:00 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/588219990' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:34:01.350 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:01 smithi150 podman[63321]: 2024-04-22 21:34:01.134208293 +0000 UTC m=+0.493260305 container cleanup e021384d6729b2adc301b98aa19981c5ae7b00e7cb411116ec5c20c28db490d5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, name=ubi8, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, version=8.5, architecture=x86_64, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , vendor=Red Hat, Inc., distribution-scope=public, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182) 2024-04-22T21:34:01.350 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:01 smithi150 bash[63321]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a 2024-04-22T21:34:01.736 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:01 smithi150 podman[63335]: 2024-04-22 21:34:01.41029452 +0000 UTC m=+0.641921261 container remove e021384d6729b2adc301b98aa19981c5ae7b00e7cb411116ec5c20c28db490d5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.openshift.expose-services=, maintainer=Paul Cuzner , vcs-type=git, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, name=ubi8) 2024-04-22T21:34:01.736 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:01 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-22T21:34:01.736 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:01 smithi150 systemd[1]: Stopped Ceph grafana.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:34:01.736 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:01 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@grafana.a.service: Consumed 5.576s CPU time. 2024-04-22T21:34:02.072 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:01 smithi150 systemd[1]: Starting Ceph grafana.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:34:02.072 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:01 smithi150 podman[63469]: 2024-04-22 21:34:01.871183589 +0000 UTC m=+0.018691734 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-04-22T21:34:02.073 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:01 smithi150 podman[63469]: 2024-04-22 21:34:01.978548977 +0000 UTC m=+0.126057147 container create 5e5fa903c0902d89394cadb2f6cdf4e77f9237736819f85d8d0c9d7ce45e5387 (image=quay.io/ceph/grafana:9.4.12, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-22T21:34:02.330 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 podman[63469]: 2024-04-22 21:34:02.197357729 +0000 UTC m=+0.344865860 container init 5e5fa903c0902d89394cadb2f6cdf4e77f9237736819f85d8d0c9d7ce45e5387 (image=quay.io/ceph/grafana:9.4.12, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-22T21:34:02.330 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 podman[63469]: 2024-04-22 21:34:02.200136395 +0000 UTC m=+0.347644529 container start 5e5fa903c0902d89394cadb2f6cdf4e77f9237736819f85d8d0c9d7ce45e5387 (image=quay.io/ceph/grafana:9.4.12, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-22T21:34:02.330 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 bash[63469]: 5e5fa903c0902d89394cadb2f6cdf4e77f9237736819f85d8d0c9d7ce45e5387 2024-04-22T21:34:02.330 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 systemd[1]: Started Ceph grafana.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:34:02.600 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.347970849Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-04-22T21:34:02.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348169445Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-22T21:34:02.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348180176Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-22T21:34:02.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348183615Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-22T21:34:02.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348186704Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-04-22T21:34:02.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348189538Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-04-22T21:34:02.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348192107Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-04-22T21:34:02.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348194834Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-04-22T21:34:02.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348198083Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-04-22T21:34:02.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348200984Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-22T21:34:02.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348203978Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-22T21:34:02.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348206487Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-22T21:34:02.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348209096Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-22T21:34:02.602 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348211828Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-22T21:34:02.602 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348214473Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-22T21:34:02.602 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.34821717Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-22T21:34:02.602 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348219985Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-22T21:34:02.602 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348222587Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-22T21:34:02.602 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=settings t=2024-04-22T21:34:02.348225805Z level=info msg="App mode production" 2024-04-22T21:34:02.602 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=sqlstore t=2024-04-22T21:34:02.348282974Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=sqlstore t=2024-04-22T21:34:02.348341242Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.361013076Z level=info msg="Starting DB migrations" 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.364319576Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.375798918Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.376933274Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.377722685Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.378950497Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.382232527Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.384573701Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.387069654Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.387585714Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.389789057Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.392425717Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-22T21:34:02.603 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.393576782Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.393941024Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.395046813Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.39612752Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.397275523Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.397650695Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.39874854Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.39985449Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.400182501Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.401470435Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.402697189Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.40323278Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.40604851Z level=info msg="Executing migration" id="create alert_image table" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.406575199Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-22T21:34:02.604 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.409143012Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-22T21:34:02.605 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.409467292Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-22T21:34:02.605 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.409901818Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-22T21:34:02.605 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.410344123Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-22T21:34:02.605 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.410441224Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-22T21:34:02.605 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.410746018Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-22T21:34:02.605 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.411260205Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-22T21:34:02.605 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.411746591Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-22T21:34:02.606 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.412074483Z level=info msg="Executing migration" id="create secrets table" 2024-04-22T21:34:02.606 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.412531729Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-22T21:34:02.606 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.418923522Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-22T21:34:02.606 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.421649494Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-22T21:34:02.606 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.42433483Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-22T21:34:02.606 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.430617515Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-22T21:34:02.606 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.436930671Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-22T21:34:02.606 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.439574809Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-22T21:34:02.606 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.44238478Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-22T21:34:02.606 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.442969227Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.445570051Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.448248525Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.45095323Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.45167927Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.452465898Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.453302542Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.454131178Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.454617018Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.455783308Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.456631282Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.457492711Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.457969914Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.458810569Z level=info msg="Executing migration" id="add correlation config column" 2024-04-22T21:34:02.607 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.460028613Z level=info msg="Executing migration" id="create entity_events table" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.460848721Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.46129921Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.461398302Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.462069136Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.462172935Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.46281259Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.463277079Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.463746699Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.464576677Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.465422408Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.466192548Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.466661921Z level=info msg="Executing migration" id="Drop public config table" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.467557521Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.468478836Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-22T21:34:02.608 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.4692986Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.469782285Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.470620527Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.474575166Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.475846982Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.477023883Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.47789548Z level=info msg="Executing migration" id="add share column" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.479148315Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.480034623Z level=info msg="Executing migration" id="create file table" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.480484475Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.481251864Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.482061839Z level=info msg="Executing migration" id="create file_meta table" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.482934808Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.483420479Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-22T21:34:02.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.486107358Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-22T21:34:02.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.488977534Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-22T21:34:02.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.491785629Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-22T21:34:02.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.492337128Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-22T21:34:02.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.495083955Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-22T21:34:02.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.498009064Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-22T21:34:02.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.500775972Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-22T21:34:02.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.501244672Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-22T21:34:02.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.504146405Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-22T21:34:02.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.506923754Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-22T21:34:02.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.509575737Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-22T21:34:02.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.510814165Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-22T21:34:02.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.513559469Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.526765554Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.529598578Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.530101549Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.534548346Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.537283548Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.539875414Z level=info msg="Executing migration" id="create folder table" 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.54030606Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.543167501Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.545865524Z level=info msg="Executing migration" id="Update folder title length" 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.548552881Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=migrator t=2024-04-22T21:34:02.549055026Z level=info msg="migrations completed" performed=105 skipped=376 duration=184.771048ms 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=sqlstore t=2024-04-22T21:34:02.54941055Z level=info msg="Created default organization" 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=plugin.finder t=2024-04-22T21:34:02.582431517Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-04-22T21:34:02.611 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=secrets t=2024-04-22T21:34:02.582529183Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-22T21:34:02.612 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=query_data t=2024-04-22T21:34:02.584776495Z level=info msg="Query Service initialization" 2024-04-22T21:34:02.612 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=live.push_http t=2024-04-22T21:34:02.587507655Z level=info msg="Live Push Gateway initialization" 2024-04-22T21:34:03.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:02 smithi130 ceph-mon[21867]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:34:03.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:02 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:03.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:02 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:03.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:02 smithi130 ceph-mon[26394]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:34:03.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:02 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:03.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:02 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:03.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-mon[22676]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:34:03.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:03.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:02 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:03.600 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=infra.usagestats.collector t=2024-04-22T21:34:03.275804577Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-22T21:34:03.600 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=provisioning.datasources t=2024-04-22T21:34:03.277899071Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-04-22T21:34:03.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=provisioning.datasources t=2024-04-22T21:34:03.277909573Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-04-22T21:34:03.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=provisioning.datasources t=2024-04-22T21:34:03.278006459Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-22T21:34:03.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=provisioning.alerting t=2024-04-22T21:34:03.301405639Z level=info msg="starting to provision alerting" 2024-04-22T21:34:03.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=provisioning.alerting t=2024-04-22T21:34:03.301419292Z level=info msg="finished to provision alerting" 2024-04-22T21:34:03.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=ngalert.state.manager t=2024-04-22T21:34:03.301506131Z level=info msg="Warming state cache for startup" 2024-04-22T21:34:03.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=ngalert.state.manager t=2024-04-22T21:34:03.301636768Z level=info msg="State cache has been initialized" states=0 duration=129.793µs 2024-04-22T21:34:03.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=ticker t=2024-04-22T21:34:03.301689819Z level=info msg=starting first_tick=2024-04-22T21:34:10Z 2024-04-22T21:34:03.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=ngalert.multiorg.alertmanager t=2024-04-22T21:34:03.301706295Z level=info msg="starting MultiOrg Alertmanager" 2024-04-22T21:34:03.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=grafanaStorageLogger t=2024-04-22T21:34:03.301798637Z level=info msg="storage starting" 2024-04-22T21:34:03.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=http.server t=2024-04-22T21:34:03.303896248Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-22T21:34:03.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=sqlstore.transactions t=2024-04-22T21:34:03.323486521Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-22T21:34:03.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=sqlstore.transactions t=2024-04-22T21:34:03.334076662Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-04-22T21:34:03.601 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:34:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=sqlstore.transactions t=2024-04-22T21:34:03.344563564Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=2 code="database is locked" 2024-04-22T21:34:04.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:04 smithi130 ceph-mon[21867]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:34:04.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:04 smithi130 ceph-mon[26394]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:34:04.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:04 smithi150 ceph-mon[22676]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:34:04.944 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:34:05.602 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:34:05.602 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (29s) 24s ago 22m 15.0M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:34:05.602 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 starting - - - - 2024-04-22T21:34:05.602 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (30s) 24s ago 21m 48.1M - 3.5 e1d6a67b021e 356bbac40ee6 2024-04-22T21:34:05.602 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (108s) 24s ago 27m 451M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:34:05.602 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (11m) 24s ago 29m 506M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:34:05.602 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (29m) 24s ago 29m 82.3M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:34:05.602 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (27m) 24s ago 27m 52.6M 2048M 17.2.0 e1d6a67b021e 861909a68cf9 2024-04-22T21:34:05.602 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (27m) 24s ago 27m 59.6M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:34:05.602 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (89s) 24s ago 23m 15.4M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:34:05.602 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (82s) 24s ago 23m 15.6M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:34:05.603 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (27m) 24s ago 27m 54.9M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:34:05.603 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (26m) 24s ago 26m 57.2M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:34:05.603 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (26m) 24s ago 26m 50.5M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:34:05.603 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (25m) 24s ago 25m 53.1M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:34:05.603 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (25m) 24s ago 25m 55.3M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:34:05.603 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (25m) 24s ago 25m 52.6M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:34:05.603 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (24m) 24s ago 24m 54.4M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:34:05.603 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (24m) 24s ago 24m 53.7M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:34:05.603 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (54s) 24s ago 23m 51.0M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:34:05.603 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (22m) 24s ago 22m 78.0M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:34:05.603 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (21m) 24s ago 21m 75.2M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:34:06.002 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:34:06.002 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:34:06.002 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T21:34:06.002 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:34:06.002 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:34:06.002 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:34:06.002 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:34:06.002 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:34:06.002 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:34:06.003 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:34:06.003 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": {}, 2024-04-22T21:34:06.003 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:34:06.003 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:34:06.003 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:34:06.003 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:34:06.003 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T21:34:06.003 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:34:06.003 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:34:06.003 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:34:06.336 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:34:06.336 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:34:06.336 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:34:06.336 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-22T21:34:06.336 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-04-22T21:34:06.336 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-04-22T21:34:06.336 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-04-22T21:34:06.336 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "2/2 daemons upgraded", 2024-04-22T21:34:06.336 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading grafana daemons", 2024-04-22T21:34:06.337 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:34:06.337 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:34:06.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:06 smithi150 ceph-mon[22676]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:34:06.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:06 smithi150 ceph-mon[22676]: from='client.15789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:06.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:06 smithi150 ceph-mon[22676]: from='client.25678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:06 smithi150 ceph-mon[22676]: from='client.25684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:06 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2566486450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:07.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:06 smithi130 ceph-mon[26394]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:34:07.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:06 smithi130 ceph-mon[26394]: from='client.15789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:07.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:06 smithi130 ceph-mon[26394]: from='client.25678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:07.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:06 smithi130 ceph-mon[26394]: from='client.25684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:07.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:06 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2566486450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:07.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:06 smithi130 ceph-mon[21867]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:34:07.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:06 smithi130 ceph-mon[21867]: from='client.15789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:07.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:06 smithi130 ceph-mon[21867]: from='client.25678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:07.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:06 smithi130 ceph-mon[21867]: from='client.25684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:07.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:06 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2566486450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:07 smithi130 ceph-mon[26394]: from='client.25696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:07 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:07 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:08.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:07 smithi130 ceph-mon[21867]: from='client.25696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:08.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:07 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:08.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:07 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:08.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:07 smithi150 ceph-mon[22676]: from='client.25696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:08.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:07 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:08.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:07 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:08.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:08 smithi150 ceph-mon[22676]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:34:08.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:08 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:08.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:08 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:08.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:08 smithi130 ceph-mon[21867]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:34:08.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:08 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:08.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:08 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:08.899 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:08 smithi130 ceph-mon[26394]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T21:34:08.899 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:08 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:08.900 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:08 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:09.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:34:08] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.43.0" 2024-04-22T21:34:11.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:10 smithi130 ceph-mon[21867]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-22T21:34:11.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:10 smithi130 ceph-mon[26394]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-22T21:34:11.072 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:10 smithi150 ceph-mon[22676]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-22T21:34:11.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:11 smithi150 ceph-mon[22676]: from='client.25657 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:11.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:11 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:11.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:11 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:11.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:11 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:11.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:11 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:34:11.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:11 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:11.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:11 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:34:11.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:11 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:34:11.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:11 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:12.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[21867]: from='client.25657 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:12.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:12.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:12.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:12.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:34:12.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:12.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:34:12.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:34:12.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:12.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[26394]: from='client.25657 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:12.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:12.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:12.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:12.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:34:12.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:12.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:34:12.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:34:12.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:11 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:12.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:12 smithi130 ceph-mon[26394]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-22T21:34:12.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:12 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:34:12.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:12 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:34:12.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:12 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:34:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:12 smithi130 ceph-mon[21867]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-22T21:34:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:12 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:34:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:12 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:34:12.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:12 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:34:13.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:12 smithi150 ceph-mon[22676]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-22T21:34:13.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:12 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:34:13.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:12 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:34:13.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:12 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:34:14.854 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-mon[26394]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T21:34:14.855 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-mon[21867]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T21:34:14.855 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:34:14.739+0000 7fdddea21700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:34:14.855 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:34:14.855 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:34:14.855 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:34:14.855 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:34:14.855 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:14.855 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:14.855 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:34:14.855 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:34:14.855 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:34:14.855 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:34:14.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:34:14.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:34:14.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:34:14.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:34:14.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:34:14.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:34:14.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:34:14.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:34:14.856 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return future.result(timeout) 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return self.__get_result() 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: raise self._exception 2024-04-22T21:34:14.857 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: use_current_daemon_image=reconfig, 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Creating ceph-iscsi config... 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: Traceback (most recent call last): 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: return _run_code(code, main_globals, None, 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:34:14.858 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: exec(code, run_globals) 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:34:14] ENGINE Bus STOPPING 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:34:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:34:14] ENGINE Bus STOPPED 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:34:14] ENGINE Bus STARTING 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:34:14] ENGINE Serving on http://:::9283 2024-04-22T21:34:14.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:14 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:34:14] ENGINE Bus STARTED 2024-04-22T21:34:15.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:14 smithi150 ceph-mon[22676]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T21:34:15.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:34:15.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T21:34:15.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:15.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:34:15.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.841 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:15.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:34:15.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:34:15.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:34:15.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:34:15.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:34:15.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:34:15.843 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:34:15.844 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T21:34:15.844 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:15.844 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:34:15.844 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.844 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.844 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.844 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.844 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.844 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.844 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.844 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.844 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:15.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:15.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:34:15.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:34:15.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:34:15.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:34:15.846 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:34:15.846 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:34:15.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:34:15.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:34:15.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:34:15.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:34:15.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:34:15.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:34:15.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:34:15.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:34:15.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:34:15.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:34:15.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:34:15.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:34:15.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:34:15.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:34:15.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:34:15.847 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:34:15.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:34:15.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:34:15.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:34:15.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:34:15.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:34:15.848 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:34:15.848 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:34:15.848 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:34:15.848 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:34:15.848 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:34:15.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:34:15.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:34:15.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:34:15.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:34:15.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:34:15.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:34:15.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:34:15.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:34:15.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:34:15.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:34:15.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:34:15.850 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:34:15.850 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:34:15.850 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:34:15.850 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:34:15.850 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:34:15.850 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:34:15.850 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.850 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.850 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:34:15.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:34:15.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:34:15.852 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:15.852 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:34:15.852 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:15.852 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:34:15.852 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:34:15.852 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:15.852 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:34:15.852 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.852 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.852 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.852 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.853 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.853 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.853 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.853 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.853 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.853 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.853 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.853 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:15.853 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:34:15.853 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:34:15.853 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:34:15.853 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:15.854 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:34:15.854 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:15.854 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:34:15.854 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:34:15.854 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:15 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:16.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:34:16.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T21:34:16.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:34:16.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:34:16.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:34:16.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:34:16.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:34:16.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:16.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:34:16.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:16.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:34:16.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:34:16.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:15 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:16.852 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[21867]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:16.852 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[21867]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:34:16.852 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:34:16.852 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:34:16.852 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:34:16.852 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:34:16.852 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:16.852 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:16.852 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:34:16.852 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:34:16.852 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:34:16.852 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:34:16.853 INFO:journalctl@ceph.mon.a.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: return future.result(timeout) 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.__get_result() 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: raise self._exception 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:34:16.854 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:34:16.855 INFO:journalctl@ceph.mon.a.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:34:16.855 INFO:journalctl@ceph.mon.a.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:34:16.855 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:16.855 INFO:journalctl@ceph.mon.a.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:16.855 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:34:16.855 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:34:16.855 INFO:journalctl@ceph.mon.a.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:34:16.855 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:34:16.855 INFO:journalctl@ceph.mon.a.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:34:16.855 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:34:16.855 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:34:16.855 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:34:16.855 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[26394]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[26394]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:34:16.856 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: return future.result(timeout) 2024-04-22T21:34:16.857 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.__get_result() 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: raise self._exception 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: Creating ceph-iscsi config... 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:16.858 INFO:journalctl@ceph.mon.c.smithi130.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:16.859 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-04-22T21:34:16.859 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:34:16.859 INFO:journalctl@ceph.mon.c.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:34:16.859 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:34:16.859 INFO:journalctl@ceph.mon.c.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:34:16.859 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:34:16.859 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:34:16.859 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:34:16.859 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:34:16.859 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:34:16.859 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:34:16.859 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.a.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.a.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[21867]: Upgrade: Finalizing container_image settings 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[21867]: Upgrade: Complete! 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[21867]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[21867]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.c.smithi130.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.c.smithi130.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:16.860 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:34:16.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T21:34:16.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[26394]: Upgrade: Finalizing container_image settings 2024-04-22T21:34:16.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[26394]: Upgrade: Complete! 2024-04-22T21:34:16.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[26394]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:34:16.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:16 smithi130 ceph-mon[26394]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:34:17.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:16 smithi150 ceph-mon[22676]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:17.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:16 smithi150 ceph-mon[22676]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:34:17.100 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:34:17.100 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:34:17.100 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:34:17.100 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:34:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:34:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:34:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:34:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:34:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:34:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:34:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:34:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:34:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:34:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:34:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2457, in _daemon_action 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: return future.result(timeout) 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T21:34:17.102 INFO:journalctl@ceph.mon.b.smithi150.stdout: return self.__get_result() 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: raise self._exception 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1377, in _create_daemon 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_current_daemon_image=reconfig, 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1692, in _run_cephadm 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: Creating ceph-iscsi config... 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: Non-zero exit code 1 from systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:17.103 INFO:journalctl@ceph.mon.b.smithi150.stdout: Traceback (most recent call last): 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: exec(code, run_globals) 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1123, in deploy_daemon 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: RuntimeError: Failed command: systemctl restart ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout: See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:34:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:16 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T21:34:17.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:16 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T21:34:17.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:16 smithi150 ceph-mon[22676]: Upgrade: Finalizing container_image settings 2024-04-22T21:34:17.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:16 smithi150 ceph-mon[22676]: Upgrade: Complete! 2024-04-22T21:34:17.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:16 smithi150 ceph-mon[22676]: Reconfiguring iscsi.foo.smithi130.pcsiph (dependencies changed)... 2024-04-22T21:34:17.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:16 smithi150 ceph-mon[22676]: Reconfiguring daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:34:17.958 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:17 smithi130 ceph-mon[21867]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:17.959 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:17 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:17.959 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:17 smithi130 ceph-mon[26394]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:17.959 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:17 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:18.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:17 smithi150 ceph-mon[22676]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:18.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:17 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:19.046 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:34:18] "GET /metrics HTTP/1.1" 200 37931 "" "Prometheus/2.43.0" 2024-04-22T21:34:19.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:19.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:19.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:34:19.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:34:19.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:34:19.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[21867]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:34:19.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1849675155' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:34:19.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:19.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:19.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:34:19.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:34:19.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:34:19.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[26394]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:34:19.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:18 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1849675155' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:34:19.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:18 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:19.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:18 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:19.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:18 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:34:19.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:18 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T21:34:19.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:18 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:34:19.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:18 smithi150 ceph-mon[22676]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T21:34:19.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:18 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1849675155' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:34:20.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:20 smithi150 ceph-mon[22676]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:34:20.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:20 smithi130 ceph-mon[26394]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:34:20.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:20 smithi130 ceph-mon[21867]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T21:34:22.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:22 smithi150 ceph-mon[22676]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:34:22.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:22 smithi130 ceph-mon[26394]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:34:22.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:22 smithi130 ceph-mon[21867]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:34:23.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:23 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:23.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:23 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:23.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:23 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:23.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:23 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:34:23.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:23 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:23.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:23 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:23.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:23 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:23.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:23 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:23.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:23 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:34:23.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:23 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:23.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:23 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:23.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:23 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:23.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:23 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:34:23.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:23 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:34:23.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:23 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:24.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:24 smithi130 ceph-mon[21867]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 1 op/s 2024-04-22T21:34:24.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:24 smithi130 ceph-mon[26394]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 1 op/s 2024-04-22T21:34:24.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:24 smithi150 ceph-mon[22676]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 1 op/s 2024-04-22T21:34:26.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:26 smithi130 ceph-mon[21867]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:34:26.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:26 smithi130 ceph-mon[26394]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:34:26.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:26 smithi150 ceph-mon[22676]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:34:27.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:27 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:34:27.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:27 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:34:27.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:27 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:34:28.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:28 smithi130 ceph-mon[21867]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:34:28.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:28 smithi130 ceph-mon[26394]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:34:28.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:28 smithi150 ceph-mon[22676]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:34:29.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:34:28] "GET /metrics HTTP/1.1" 200 37929 "" "Prometheus/2.43.0" 2024-04-22T21:34:29.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:29 smithi130 ceph-mon[26394]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:29.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:29 smithi130 ceph-mon[21867]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:29.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:29 smithi150 ceph-mon[22676]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:30.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:30 smithi150 ceph-mon[22676]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 1 op/s 2024-04-22T21:34:30.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:30 smithi130 ceph-mon[26394]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 1 op/s 2024-04-22T21:34:30.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:30 smithi130 ceph-mon[21867]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 1 op/s 2024-04-22T21:34:32.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:32 smithi130 ceph-mon[21867]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:32.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:32 smithi130 ceph-mon[26394]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:32.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:32 smithi150 ceph-mon[22676]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:34.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:34 smithi130 ceph-mon[21867]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:34.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:34 smithi130 ceph-mon[26394]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:34.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:34 smithi150 ceph-mon[22676]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:36.699 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:36 smithi130 ceph-mon[21867]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:36.699 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:36 smithi130 ceph-mon[26394]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:36.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:36 smithi150 ceph-mon[22676]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:37.304 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-04-22T21:34:38.691 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:38 smithi130 ceph-mon[26394]: from='client.25726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:38.691 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:38 smithi130 ceph-mon[26394]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:38.691 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:38 smithi130 ceph-mon[21867]: from='client.25726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:38.691 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:38 smithi130 ceph-mon[21867]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:38.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:38 smithi150 ceph-mon[22676]: from='client.25726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:38.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:38 smithi150 ceph-mon[22676]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:39.028 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:34:38] "GET /metrics HTTP/1.1" 200 37929 "" "Prometheus/2.43.0" 2024-04-22T21:34:39.437 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:34:39.748 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:39 smithi130 ceph-mon[21867]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:39.748 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:39 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/1076021668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:39.748 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:39 smithi130 ceph-mon[26394]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:39.748 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:39 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/1076021668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:39.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:39 smithi150 ceph-mon[22676]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:39.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:39 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/1076021668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:40.072 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-04-22T21:34:40.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:40 smithi130 ceph-mon[21867]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:40.702 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:40 smithi130 ceph-mon[26394]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:40.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:40 smithi150 ceph-mon[22676]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:42.132 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)" 2024-04-22T21:34:42.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:42 smithi130 ceph-mon[21867]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:42.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:42 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:34:42.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:42 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4217742144' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:42.747 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:42 smithi130 ceph-mon[26394]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:42.747 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:42 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:34:42.748 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:42 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4217742144' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:42.771 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-04-22T21:34:42.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:42 smithi150 ceph-mon[22676]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:42.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:42 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:34:42.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:42 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4217742144' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:44.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:44 smithi130 ceph-mon[21867]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:44.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:44 smithi130 ceph-mon[26394]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:44.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:44 smithi150 ceph-mon[22676]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:44.872 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:34:45.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-04-22T21:34:45.772 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:45 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/2822830785' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:45.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:45 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/2822830785' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:45.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:45 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/2822830785' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:34:46.637 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:46 smithi130 ceph-mon[21867]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:46.637 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:46 smithi130 ceph-mon[26394]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:46.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:46 smithi150 ceph-mon[22676]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:48.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:48 smithi130 ceph-mon[26394]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:48.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:48 smithi130 ceph-mon[26394]: from='client.15867 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:48.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:48 smithi130 ceph-mon[21867]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:48.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:48 smithi130 ceph-mon[21867]: from='client.15867 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:48.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:48 smithi150 ceph-mon[22676]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:48.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:48 smithi150 ceph-mon[22676]: from='client.15867 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:49.006 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:34:49.178 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:34:48] "GET /metrics HTTP/1.1" 200 37931 "" "Prometheus/2.43.0" 2024-04-22T21:34:49.609 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade status' 2024-04-22T21:34:49.753 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:49 smithi150 ceph-mon[22676]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:49.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:49 smithi130 ceph-mon[21867]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:49.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:49 smithi130 ceph-mon[26394]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:50.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:50 smithi130 ceph-mon[21867]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:50.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:50 smithi130 ceph-mon[26394]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:50.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:50 smithi150 ceph-mon[22676]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:51.626 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:34:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": null, 2024-04-22T21:34:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": false, 2024-04-22T21:34:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "", 2024-04-22T21:34:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:34:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": null, 2024-04-22T21:34:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "", 2024-04-22T21:34:51.626 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:34:51.626 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:34:52.292 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph health detail' 2024-04-22T21:34:52.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:52 smithi130 ceph-mon[26394]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:52.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:52 smithi130 ceph-mon[26394]: from='client.15873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:52.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:52 smithi130 ceph-mon[21867]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:52.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:52 smithi130 ceph-mon[21867]: from='client.15873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:52.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:52 smithi150 ceph-mon[22676]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:52.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:52 smithi150 ceph-mon[22676]: from='client.15873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:54.442 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-04-22T21:34:54.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:54 smithi130 ceph-mon[21867]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:54.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:54 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/3258160647' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:34:54.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:54 smithi130 ceph-mon[26394]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:54.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:54 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/3258160647' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:34:54.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:54 smithi150 ceph-mon[22676]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:34:54.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:54 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/3258160647' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:34:55.073 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-04-22T21:34:56.569 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:56 smithi130 ceph-mon[26394]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:56.569 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:56 smithi130 ceph-mon[21867]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:56.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:56 smithi150 ceph-mon[22676]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:57.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:57 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:34:57.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:57 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:34:57.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:57 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:34:58.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:58 smithi130 ceph-mon[21867]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:58.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:58 smithi130 ceph-mon[21867]: from='client.25768 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:58.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:58 smithi130 ceph-mon[21867]: from='client.15891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "mon", "hosts": "smithi150", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:58.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:58 smithi130 ceph-mon[26394]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:58.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:58 smithi130 ceph-mon[26394]: from='client.25768 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:58.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:58 smithi130 ceph-mon[26394]: from='client.15891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "mon", "hosts": "smithi150", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:58.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:58 smithi150 ceph-mon[22676]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:34:58.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:58 smithi150 ceph-mon[22676]: from='client.25768 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:58.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:58 smithi150 ceph-mon[22676]: from='client.15891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "mon", "hosts": "smithi150", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:34:58.900 INFO:teuthology.orchestra.run.smithi130.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:34:59.230 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:34:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:34:58] "GET /metrics HTTP/1.1" 200 37926 "" "Prometheus/2.43.0" 2024-04-22T21:34:59.522 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:59 smithi130 ceph-mon[26394]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:59.523 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:34:59 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:59.523 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:59 smithi130 ceph-mon[21867]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:59.523 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:34:59 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:34:59.546 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T21:34:59.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:59 smithi150 ceph-mon[22676]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:34:59.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:34:59 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:00.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:00 smithi130 ceph-mon[26394]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:00.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:00 smithi130 ceph-mon[26394]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:00.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:00 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:00.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:00 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:00.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:00 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:00.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:00 smithi130 ceph-mon[21867]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:00.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:00 smithi130 ceph-mon[21867]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:00.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:00 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:00.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:00 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:00.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:00 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:00.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:00 smithi150 ceph-mon[22676]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:00.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:00 smithi150 ceph-mon[22676]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:00.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:00 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:00.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:00 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:00.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:00 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:01.542 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:35:01.746 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:01 smithi130 ceph-mon[21867]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:01.747 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:01 smithi130 ceph-mon[26394]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:01.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:01 smithi150 ceph-mon[22676]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:02.215 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:35:02.215 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (86s) 39s ago 23m 26.5M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:35:02.215 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (60s) 55s ago 23m 49.1M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:35:02.215 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (44s) 39s ago 22m 47.9M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (2m) 55s ago 28m 452M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (12m) 39s ago 30m 514M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (30m) 39s ago 30m 83.1M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (28m) 55s ago 28m 53.6M 2048M 17.2.0 e1d6a67b021e 861909a68cf9 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (28m) 39s ago 28m 60.8M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (2m) 39s ago 24m 16.0M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (2m) 55s ago 24m 15.8M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (28m) 39s ago 28m 54.9M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (27m) 39s ago 27m 57.2M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (27m) 39s ago 27m 50.5M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (26m) 39s ago 26m 53.1M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (26m) 55s ago 26m 55.3M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (26m) 55s ago 26m 52.6M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:35:02.216 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (25m) 55s ago 25m 54.4M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:35:02.217 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (25m) 55s ago 25m 53.7M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:35:02.217 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (111s) 55s ago 24m 51.5M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:35:02.217 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (22m) 39s ago 22m 78.1M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:35:02.217 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (22m) 55s ago 22m 75.2M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:35:02.639 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:35:02.639 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:35:02.639 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T21:35:02.639 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": {}, 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:35:02.640 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:35:02.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[26394]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:02.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[26394]: from='client.15897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:02.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:02.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:35:02.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:02.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:02.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T21:35:02.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-22T21:35:02.642 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[21867]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:02.642 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[21867]: from='client.15897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:02.642 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:02.642 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:35:02.642 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:02.642 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:02.642 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T21:35:02.642 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:02 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-22T21:35:02.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:02 smithi150 ceph-mon[22676]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:02.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:02 smithi150 ceph-mon[22676]: from='client.15897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:02.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:02 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:02.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:02 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:35:02.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:02 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:02.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:02 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:02.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:02 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T21:35:02.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:02 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-22T21:35:02.971 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:35:02.971 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:35:02.971 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:35:02.971 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi150", 2024-04-22T21:35:02.972 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:35:02.972 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "0/1 daemons upgraded", 2024-04-22T21:35:02.972 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading mon daemons", 2024-04-22T21:35:02.972 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:35:02.972 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:35:03.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:03 smithi150 ceph-mon[22676]: from='client.15903 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:03.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:03 smithi150 ceph-mon[22676]: from='client.25792 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:03.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:03 smithi150 ceph-mon[22676]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:35:03.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:03 smithi150 ceph-mon[22676]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:35:03.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:03 smithi150 ceph-mon[22676]: Upgrade: Setting container_image for all mgr 2024-04-22T21:35:03.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:03 smithi150 ceph-mon[22676]: Upgrade: It appears safe to stop mon.b 2024-04-22T21:35:03.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:03 smithi150 ceph-mon[22676]: from='client.? 172.21.15.130:0/4114962975' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:03.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[26394]: from='client.15903 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:03.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[26394]: from='client.25792 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:03.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[26394]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:35:03.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[26394]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:35:03.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all mgr 2024-04-22T21:35:03.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[26394]: Upgrade: It appears safe to stop mon.b 2024-04-22T21:35:03.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/4114962975' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:03.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[21867]: from='client.15903 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:03.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[21867]: from='client.25792 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:03.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[21867]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:35:03.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[21867]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:35:03.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all mgr 2024-04-22T21:35:03.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[21867]: Upgrade: It appears safe to stop mon.b 2024-04-22T21:35:03.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:03 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/4114962975' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:04.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:04 smithi130 ceph-mon[26394]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:04.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:04 smithi130 ceph-mon[26394]: from='client.25804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:04.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:04 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:04.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:04 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:35:04.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:04 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:35:04.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:04 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:04.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:04 smithi130 ceph-mon[21867]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:04.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:04 smithi130 ceph-mon[21867]: from='client.25804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:04.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:04 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:04.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:04 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:35:04.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:04 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:35:04.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:04 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:04.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:04 smithi150 ceph-mon[22676]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:04.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:04 smithi150 ceph-mon[22676]: from='client.25804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:04.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:04 smithi150 ceph-mon[22676]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:04.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:04 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:35:04.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:04 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:35:04.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:04 smithi150 ceph-mon[22676]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:05.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:05 smithi130 ceph-mon[21867]: Upgrade: Updating mon.b 2024-04-22T21:35:05.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:05 smithi130 ceph-mon[21867]: Deploying daemon mon.b on smithi150 2024-04-22T21:35:05.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:05 smithi130 ceph-mon[26394]: Upgrade: Updating mon.b 2024-04-22T21:35:05.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:05 smithi130 ceph-mon[26394]: Deploying daemon mon.b on smithi150 2024-04-22T21:35:05.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:05 smithi150 ceph-mon[22676]: Upgrade: Updating mon.b 2024-04-22T21:35:05.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:05 smithi150 ceph-mon[22676]: Deploying daemon mon.b on smithi150 2024-04-22T21:35:06.344 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:06 smithi150 systemd[1]: Stopping Ceph mon.b for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:35:06.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b[22671]: 2024-04-22T21:35:06.360+0000 7fe90d498700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:35:06.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b[22671]: 2024-04-22T21:35:06.360+0000 7fe90d498700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-04-22T21:35:06.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:06 smithi150 podman[64939]: 2024-04-22 21:35:06.534700983 +0000 UTC m=+0.289320281 container died 861909a68cf92d263def712ac54b5359bddb0a9aad280c48be1096b13c849172 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b, GIT_CLEAN=True, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, architecture=x86_64, name=centos-stream, ceph=True, io.openshift.tags=base centos centos-stream, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-22T21:35:07.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:06 smithi150 podman[64939]: 2024-04-22 21:35:06.902331922 +0000 UTC m=+0.656951220 container cleanup 861909a68cf92d263def712ac54b5359bddb0a9aad280c48be1096b13c849172 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.openshift.tags=base centos centos-stream, distribution-scope=public, RELEASE=HEAD, architecture=x86_64, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, release=754, ceph=True) 2024-04-22T21:35:07.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:06 smithi150 bash[64939]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b 2024-04-22T21:35:07.436 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 podman[64953]: 2024-04-22 21:35:07.146251222 +0000 UTC m=+0.612966659 container remove 861909a68cf92d263def712ac54b5359bddb0a9aad280c48be1096b13c849172 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b, GIT_CLEAN=True, ceph=True, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, release=754, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., architecture=x86_64, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, vcs-type=git) 2024-04-22T21:35:07.436 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-22T21:35:07.436 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 systemd[1]: Stopped Ceph mon.b for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:35:07.437 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.b.service: Consumed 18.148s CPU time. 2024-04-22T21:35:07.792 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 systemd[1]: Starting Ceph mon.b for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:35:07.792 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 podman[65083]: 2024-04-22 21:35:07.574468887 +0000 UTC m=+0.018382019 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:07.792 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 podman[65083]: 2024-04-22 21:35:07.704916135 +0000 UTC m=+0.148829252 container create 241734f80ee17280373c5cf45aa3d3de3896576eb62a6308926551e512fdb00d (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b, maintainer=Guillaume Abrioux , RELEASE=main-56e81df, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=) 2024-04-22T21:35:08.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 podman[65083]: 2024-04-22 21:35:07.932388563 +0000 UTC m=+0.376301681 container init 241734f80ee17280373c5cf45aa3d3de3896576eb62a6308926551e512fdb00d (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T21:35:08.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 podman[65083]: 2024-04-22 21:35:07.935399331 +0000 UTC m=+0.379312451 container start 241734f80ee17280373c5cf45aa3d3de3896576eb62a6308926551e512fdb00d (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, RELEASE=main-56e81df, org.label-schema.build-date=20240416) 2024-04-22T21:35:08.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T21:35:08.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev), process ceph-mon, pid 2 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: pidfile_write: ignore empty --pid-file 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: load: jerasure load: lrc 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: RocksDB version: 7.9.2 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Git sha 0 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Compile date 2024-04-22 15:21:46 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: DB SUMMARY 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: DB Session ID: IPNY2JYF11SOJR4NQKB9 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: CURRENT file: CURRENT 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2829 Bytes 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 3769616 ; 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.error_if_exists: 0 2024-04-22T21:35:08.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.create_if_missing: 0 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.paranoid_checks: 1 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.env: 0x564be3d5bcc0 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.fs: PosixFileSystem 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.info_log: 0x564be64c9be0 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.statistics: (nil) 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.use_fsync: 0 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_log_file_size: 0 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T21:35:08.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.allow_fallocate: 1 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.use_direct_reads: 0 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.db_log_dir: 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.wal_dir: 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T21:35:08.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.write_buffer_manager: 0x564be645d220 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.unordered_write: 0 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T21:35:08.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.row_cache: None 2024-04-22T21:35:08.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.wal_filter: None 2024-04-22T21:35:08.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T21:35:08.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T21:35:08.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.two_write_queues: 0 2024-04-22T21:35:08.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T21:35:08.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.wal_compression: 0 2024-04-22T21:35:08.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.atomic_flush: 0 2024-04-22T21:35:08.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T21:35:08.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T21:35:08.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T21:35:08.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.log_readahead_size: 0 2024-04-22T21:35:08.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T21:35:08.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_background_jobs: 2 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_background_compactions: -1 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_subcompactions: 1 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T21:35:08.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_open_files: -1 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_background_flushes: -1 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Compression algorithms supported: 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: kZSTD supported: 0 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: kXpressCompression supported: 0 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: kZlibCompression supported: 1 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: kSnappyCompression supported: 1 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: kLZ4Compression supported: 1 2024-04-22T21:35:08.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: kBZip2Compression supported: 0 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.merge_operator: 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compaction_filter: None 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564be64c9dc0) 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout: cache_index_and_filter_blocks: 1 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout: pin_top_level_index_and_filter: 1 2024-04-22T21:35:08.109 INFO:journalctl@ceph.mon.b.smithi150.stdout: index_type: 0 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: data_block_index_type: 0 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: index_shortening: 1 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: checksum: 4 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: no_block_cache: 0 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache: 0x564be648f1f0 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache_name: BinnedLRUCache 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache_options: 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: capacity : 536870912 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: num_shard_bits : 4 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: strict_capacity_limit : 0 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: high_pri_pool_ratio: 0.000 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache_compressed: (nil) 2024-04-22T21:35:08.110 INFO:journalctl@ceph.mon.b.smithi150.stdout: persistent_cache: (nil) 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_size: 4096 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_size_deviation: 10 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_restart_interval: 16 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: index_block_restart_interval: 1 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: metadata_block_size: 4096 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: partition_filters: 0 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_delta_encoding: 1 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: filter_policy: bloomfilter 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: whole_key_filtering: 1 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: verify_compression: 0 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: read_amp_bytes_per_bit: 0 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: format_version: 5 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: enable_index_compression: 1 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_align: 0 2024-04-22T21:35:08.111 INFO:journalctl@ceph.mon.b.smithi150.stdout: max_auto_readahead_size: 262144 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout: prepopulate_block_cache: 0 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout: initial_auto_readahead_size: 8192 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout: num_file_reads_for_auto_readahead: 2 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compression: NoCompression 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.num_levels: 7 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T21:35:08.112 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-22T21:35:08.113 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T21:35:08.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.arena_block_size: 1048576 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T21:35:08.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.inplace_update_support: 0 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.bloom_locality: 0 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.max_successive_merges: 0 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.ttl: 2592000 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T21:35:08.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.enable_blob_files: false 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.min_blob_size: 0 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.blob_file_starting_level: 0 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28836, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-04-22T21:35:08.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 0993097f-d96d-4563-82b8-d8c336ee647f 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713821707965281, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713821707974636, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 2311314, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28841, "largest_seqno": 31040, "table_properties": {"data_size": 2303102, "index_size": 5118, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2053, "raw_key_size": 19389, "raw_average_key_size": 23, "raw_value_size": 2285870, "raw_average_value_size": 2801, "num_data_blocks": 231, "num_entries": 816, "num_filter_entries": 816, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1713821707, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "0993097f-d96d-4563-82b8-d8c336ee647f", "db_session_id": "IPNY2JYF11SOJR4NQKB9", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713821707974737, "job": 1, "event": "recovery_finished"} 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x564be65a4000 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: DB pointer 0x564be658e000 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** DB Stats ** 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T21:35:08.118 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: L0 1/0 2.20 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 580.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: L6 1/0 9.32 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: Sum 2/0 11.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 580.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 580.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T21:35:08.119 INFO:journalctl@ceph.mon.b.smithi150.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 580.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative compaction: 0.00 GB write, 159.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval compaction: 0.00 GB write, 159.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: Block cache BinnedLRUCache@0x564be648f1f0#2 capacity: 512.00 MB usage: 7.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-22T21:35:08.120 INFO:journalctl@ceph.mon.b.smithi150.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.28 KB,0.000435114%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: starting mon.b rank 2 at public addrs [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] at bind addrs [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: mon.b@-1(???) e3 preinit fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: mon.b@-1(???).mds e1 new map 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: mon.b@-1(???).mds e1 print_map 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout: e1 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout: legacy client fscid: -1 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout: No filesystems configured 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: mon.b@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T21:35:08.121 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:07 smithi150 ceph-mon[65103]: mon.b@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-04-22T21:35:08.122 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:08 smithi150 bash[65083]: 241734f80ee17280373c5cf45aa3d3de3896576eb62a6308926551e512fdb00d 2024-04-22T21:35:08.122 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:08 smithi150 systemd[1]: Started Ceph mon.b for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:35:08.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:08 smithi130 ceph-mon[21867]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:08.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:08 smithi130 ceph-mon[26394]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:08.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:08 smithi150 ceph-mon[65103]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:09.181 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:35:08] "GET /metrics HTTP/1.1" 200 37926 "" "Prometheus/2.43.0" 2024-04-22T21:35:09.299 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:09 smithi150 ceph-mon[65103]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:09.299 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:09 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:09.299 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:09 smithi150 ceph-mon[65103]: mon.b calling monitor election 2024-04-22T21:35:09.299 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:09 smithi150 ceph-mon[65103]: mon.c calling monitor election 2024-04-22T21:35:09.299 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:09 smithi150 ceph-mon[65103]: mon.a calling monitor election 2024-04-22T21:35:09.299 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:09 smithi150 ceph-mon[65103]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T21:35:09.299 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:09 smithi150 ceph-mon[65103]: monmap e3: 3 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} 2024-04-22T21:35:09.299 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:09 smithi150 ceph-mon[65103]: fsmap 2024-04-22T21:35:09.299 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:09 smithi150 ceph-mon[65103]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T21:35:09.299 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:09 smithi150 ceph-mon[65103]: mgrmap e43: y(active, since 3m), standbys: x 2024-04-22T21:35:09.299 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:09 smithi150 ceph-mon[65103]: overall HEALTH_OK 2024-04-22T21:35:09.299 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:09 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:09.300 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:09 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:09.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[26394]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:09.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[26394]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:09.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[26394]: mon.b calling monitor election 2024-04-22T21:35:09.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[26394]: mon.c calling monitor election 2024-04-22T21:35:09.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[26394]: mon.a calling monitor election 2024-04-22T21:35:09.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[26394]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T21:35:09.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[26394]: monmap e3: 3 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} 2024-04-22T21:35:09.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[26394]: fsmap 2024-04-22T21:35:09.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[26394]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T21:35:09.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[26394]: mgrmap e43: y(active, since 3m), standbys: x 2024-04-22T21:35:09.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[26394]: overall HEALTH_OK 2024-04-22T21:35:09.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:09.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:09.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[21867]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:09.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[21867]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:09.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[21867]: mon.b calling monitor election 2024-04-22T21:35:09.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[21867]: mon.c calling monitor election 2024-04-22T21:35:09.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[21867]: mon.a calling monitor election 2024-04-22T21:35:09.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[21867]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T21:35:09.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[21867]: monmap e3: 3 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} 2024-04-22T21:35:09.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[21867]: fsmap 2024-04-22T21:35:09.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[21867]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T21:35:09.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[21867]: mgrmap e43: y(active, since 3m), standbys: x 2024-04-22T21:35:09.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[21867]: overall HEALTH_OK 2024-04-22T21:35:09.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:09.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:09 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:10.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:10 smithi150 ceph-mon[65103]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:10.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:10 smithi130 ceph-mon[21867]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:10.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:10 smithi130 ceph-mon[26394]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:12.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:12 smithi150 ceph-mon[65103]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:12.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:12 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:35:12.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:12 smithi130 ceph-mon[26394]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:12.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:12 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:35:12.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:12 smithi130 ceph-mon[21867]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:12.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:12 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:35:13.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:12 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:35:12.984+0000 7fdde1a27700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-22T21:35:14.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:13 smithi130 ceph-mon[26394]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:14.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:13 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:14.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:13 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:14.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:13 smithi130 ceph-mon[21867]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:14.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:13 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:14.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:13 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:14.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:13 smithi150 ceph-mon[65103]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:14.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:13 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:14.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:13 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:15.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:14 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:15.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:14 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:15.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:14 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:15.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:14 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:15.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:14 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:15.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:14 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:16.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:16 smithi150 ceph-mon[65103]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:16.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:16 smithi130 ceph-mon[21867]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:16.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:16 smithi130 ceph-mon[26394]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:18.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:18.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Detected new or changed devices on smithi150 2024-04-22T21:35:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:35:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all crash 2024-04-22T21:35:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mds 2024-04-22T21:35:18.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all nfs 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all node-exporter 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all prometheus 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all alertmanager 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all grafana 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all loki 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all promtail 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Finalizing container_image settings 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:35:18.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:35:18.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:35:18.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:35:18.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:35:18.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:35:18.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:35:18.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:35:18.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:35:18.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:35:18.606 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:35:18.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:35:18.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:35:18.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:35:18.607 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:35:18.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.608 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: Upgrade: Complete! 2024-04-22T21:35:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:35:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:35:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:35:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:18.609 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:18 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:18.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Detected new or changed devices on smithi150 2024-04-22T21:35:18.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all crash 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all mds 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T21:35:18.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all nfs 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all node-exporter 2024-04-22T21:35:18.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:18.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Detected new or changed devices on smithi150 2024-04-22T21:35:18.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:18.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:18.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:35:18.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all crash 2024-04-22T21:35:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all mds 2024-04-22T21:35:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:35:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T21:35:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-22T21:35:18.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:35:18.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all nfs 2024-04-22T21:35:18.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:35:18.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all node-exporter 2024-04-22T21:35:18.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all prometheus 2024-04-22T21:35:18.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all alertmanager 2024-04-22T21:35:18.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all grafana 2024-04-22T21:35:18.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all loki 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all promtail 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Finalizing container_image settings 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all prometheus 2024-04-22T21:35:18.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all alertmanager 2024-04-22T21:35:18.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all grafana 2024-04-22T21:35:18.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all loki 2024-04-22T21:35:18.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:18.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all promtail 2024-04-22T21:35:18.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T21:35:18.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Finalizing container_image settings 2024-04-22T21:35:18.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:35:18.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:35:18.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:35:18.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:35:18.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:35:18.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:35:18.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:35:18.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:35:18.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:35:18.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:35:18.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:35:18.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:35:18.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:35:18.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:35:18.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:35:18.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:35:18.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:35:18.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:35:18.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:35:18.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:35:18.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:35:18.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:35:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:35:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:35:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:35:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:35:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:35:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:35:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:35:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:35:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:35:18.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:35:18.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: Upgrade: Complete! 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:35:18.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: Upgrade: Complete! 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:35:18.812 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:35:18.813 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:18.813 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:18.813 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:18.813 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:18.813 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:18.813 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:19.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:35:18] "GET /metrics HTTP/1.1" 200 37872 "" "Prometheus/2.43.0" 2024-04-22T21:35:19.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:19 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:19.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:19 smithi130 ceph-mon[26394]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:19.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:19 smithi130 ceph-mon[21867]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:20.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:20 smithi150 ceph-mon[65103]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:20.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:20 smithi130 ceph-mon[21867]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:20.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:20 smithi130 ceph-mon[26394]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:22.598 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:22 smithi130 ceph-mon[21867]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:22.598 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:22 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:22.598 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:22 smithi130 ceph-mon[26394]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:22.598 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:22 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:22.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:22 smithi150 ceph-mon[65103]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:22.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:22 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:24.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:24 smithi150 ceph-mon[65103]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:24.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:24 smithi130 ceph-mon[26394]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:24.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:24 smithi130 ceph-mon[21867]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:26.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:26 smithi150 ceph-mon[65103]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:26.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:26 smithi130 ceph-mon[26394]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:26.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:26 smithi130 ceph-mon[21867]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:27.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:27 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:35:27.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:27 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:35:27.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:27 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:35:28.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:28 smithi150 ceph-mon[65103]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:28.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:28 smithi130 ceph-mon[21867]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:28.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:28 smithi130 ceph-mon[26394]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:29.296 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:35:28] "GET /metrics HTTP/1.1" 200 37941 "" "Prometheus/2.43.0" 2024-04-22T21:35:29.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:29 smithi130 ceph-mon[26394]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:29.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:29 smithi130 ceph-mon[21867]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:29.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:29 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:30.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:30 smithi130 ceph-mon[21867]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:30.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:30 smithi130 ceph-mon[26394]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:30.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:30 smithi150 ceph-mon[65103]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:32.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:32 smithi130 ceph-mon[21867]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:32.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:32 smithi130 ceph-mon[26394]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:32.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:32 smithi150 ceph-mon[65103]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:34.037 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps' 2024-04-22T21:35:34.658 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:34 smithi130 ceph-mon[26394]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:34.658 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:34 smithi130 ceph-mon[26394]: from='client.25810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:34.659 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:34 smithi130 ceph-mon[21867]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:34.659 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:34 smithi130 ceph-mon[21867]: from='client.25810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:34.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:34 smithi150 ceph-mon[65103]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:34.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:34 smithi150 ceph-mon[65103]: from='client.25810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:35.973 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:35:35.973 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (2m) 73s ago 24m 26.5M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:35:35.973 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (94s) 23s ago 24m 76.4M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:35:35.973 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (78s) 73s ago 23m 47.9M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:35:35.973 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (3m) 23s ago 28m 452M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:35:35.973 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (13m) 73s ago 30m 514M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:35:35.973 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (30m) 73s ago 30m 83.1M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:35:35.973 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (28s) 23s ago 29m 19.6M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:35:35.973 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (29m) 73s ago 29m 60.8M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:35:35.974 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (2m) 73s ago 25m 16.0M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:35:35.974 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (2m) 23s ago 24m 15.5M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:35:35.974 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (28m) 73s ago 28m 54.9M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:35:35.974 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (28m) 73s ago 28m 57.2M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:35:35.974 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (27m) 73s ago 27m 50.5M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:35:35.974 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (27m) 73s ago 27m 53.1M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:35:35.974 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (27m) 23s ago 27m 55.6M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:35:35.974 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (26m) 23s ago 26m 52.7M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:35:35.974 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (26m) 23s ago 26m 54.4M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:35:35.974 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (25m) 23s ago 25m 53.7M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:35:35.974 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (2m) 23s ago 25m 65.1M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:35:35.974 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (23m) 73s ago 23m 78.1M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:35:35.975 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (23m) 23s ago 23m 75.9M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:35:36.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:36 smithi130 ceph-mon[21867]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:36.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:36 smithi130 ceph-mon[26394]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:36.603 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-04-22T21:35:36.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:36 smithi150 ceph-mon[65103]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:37.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:37 smithi130 ceph-mon[21867]: from='client.25816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:37.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:37 smithi130 ceph-mon[26394]: from='client.25816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:37.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:37 smithi150 ceph-mon[65103]: from='client.25816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:38.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:38 smithi130 ceph-mon[26394]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:38.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:38 smithi130 ceph-mon[21867]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:38.716 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:35:38.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:38 smithi150 ceph-mon[65103]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:39.046 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:35:38] "GET /metrics HTTP/1.1" 200 37941 "" "Prometheus/2.43.0" 2024-04-22T21:35:39.469 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:39 smithi130 ceph-mon[21867]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:39.469 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:39 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/150971235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:39.469 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:39 smithi130 ceph-mon[26394]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:39.469 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:39 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/150971235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:39.493 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade status' 2024-04-22T21:35:39.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:39 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:39.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:39 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/150971235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:40.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:40 smithi130 ceph-mon[26394]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:40.759 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:40 smithi130 ceph-mon[21867]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:40.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:40 smithi150 ceph-mon[65103]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:41.437 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:35:41.437 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": null, 2024-04-22T21:35:41.437 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": false, 2024-04-22T21:35:41.437 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "", 2024-04-22T21:35:41.437 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:35:41.437 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": null, 2024-04-22T21:35:41.438 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "", 2024-04-22T21:35:41.438 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:35:41.438 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:35:42.132 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph health detail' 2024-04-22T21:35:42.655 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:42 smithi130 ceph-mon[21867]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:42.655 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:42 smithi130 ceph-mon[21867]: from='client.15945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:42.655 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:42 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:35:42.655 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:42 smithi130 ceph-mon[26394]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:42.655 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:42 smithi130 ceph-mon[26394]: from='client.15945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:42.655 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:42 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:35:42.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:42 smithi150 ceph-mon[65103]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:42.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:42 smithi150 ceph-mon[65103]: from='client.15945 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:42.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:42 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:35:44.146 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-04-22T21:35:44.457 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:44 smithi130 ceph-mon[26394]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:44.458 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:44 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/254983491' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:35:44.458 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:44 smithi130 ceph-mon[21867]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:44.458 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:44 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/254983491' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:35:44.766 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-04-22T21:35:44.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:44 smithi150 ceph-mon[65103]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:44.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:44 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/254983491' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:35:46.636 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:46 smithi130 ceph-mon[21867]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:46.636 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:46 smithi130 ceph-mon[26394]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:46.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:46 smithi150 ceph-mon[65103]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:48.536 INFO:teuthology.orchestra.run.smithi130.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:48.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:48 smithi130 ceph-mon[21867]: from='client.15957 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:48.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:48 smithi130 ceph-mon[21867]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:48.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:48 smithi130 ceph-mon[21867]: from='client.25846 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "mon", "hosts": "smithi130", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:48.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:48 smithi130 ceph-mon[26394]: from='client.15957 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:48.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:48 smithi130 ceph-mon[26394]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:48.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:48 smithi130 ceph-mon[26394]: from='client.25846 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "mon", "hosts": "smithi130", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:48.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:48 smithi150 ceph-mon[65103]: from='client.15957 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:48.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:48 smithi150 ceph-mon[65103]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:48.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:48 smithi150 ceph-mon[65103]: from='client.25846 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "mon", "hosts": "smithi130", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:49.223 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:35:48] "GET /metrics HTTP/1.1" 200 37939 "" "Prometheus/2.43.0" 2024-04-22T21:35:49.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T21:35:49.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:49 smithi130 ceph-mon[21867]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:49.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:49 smithi130 ceph-mon[21867]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:49.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:49 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:49.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:49.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:49 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:49.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:49 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:49.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:49 smithi130 ceph-mon[26394]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:49.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:49 smithi130 ceph-mon[26394]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:49.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:49 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:49.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:49.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:49 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:49.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:49 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:49.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:49 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:35:49.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:49 smithi150 ceph-mon[65103]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:49.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:49 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:49.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:49 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:49.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:49 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:35:49.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:49 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:50.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:50 smithi150 ceph-mon[65103]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:50.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:50 smithi150 ceph-mon[65103]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:50.926 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:50 smithi130 ceph-mon[21867]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:50.926 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:50 smithi130 ceph-mon[21867]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:50.927 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:50 smithi130 ceph-mon[26394]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:50.927 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:50 smithi130 ceph-mon[26394]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:52.138 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:35:52.648 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[26394]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:52.648 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:52.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[26394]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:35:52.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[26394]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:35:52.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:35:52.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:52.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[26394]: Upgrade: Setting container_image for all mgr 2024-04-22T21:35:52.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:52.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T21:35:52.840 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:35:52.840 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (2m) 90s ago 24m 26.5M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:35:52.840 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (110s) 39s ago 24m 76.4M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (95s) 90s ago 23m 47.9M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (3m) 39s ago 29m 452M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (13m) 90s ago 30m 514M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (30m) 90s ago 30m 83.1M 2048M 17.2.0 e1d6a67b021e e010afa4af4d 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (45s) 39s ago 29m 19.6M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (29m) 90s ago 29m 60.8M 2048M 17.2.0 e1d6a67b021e 0e0f9de395ab 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (3m) 90s ago 25m 16.0M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (3m) 39s ago 25m 15.5M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (28m) 90s ago 28m 54.9M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (28m) 90s ago 28m 57.2M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (28m) 90s ago 28m 50.5M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (27m) 90s ago 27m 53.1M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (27m) 39s ago 27m 55.6M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (26m) 39s ago 26m 52.7M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:35:52.841 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (26m) 39s ago 26m 54.4M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:35:52.842 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (26m) 39s ago 26m 53.7M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:35:52.842 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (2m) 39s ago 25m 65.1M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:35:52.842 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (23m) 90s ago 23m 78.1M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:35:52.842 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (23m) 39s ago 23m 75.9M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:35:52.943 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-22T21:35:52.944 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[21867]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:52.944 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:52.944 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[21867]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:35:52.944 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[21867]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:35:52.944 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:35:52.944 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:52.944 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[21867]: Upgrade: Setting container_image for all mgr 2024-04-22T21:35:52.944 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:52.944 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T21:35:52.944 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:52 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-22T21:35:53.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:52 smithi150 ceph-mon[65103]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:53.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:52 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:53.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:52 smithi150 ceph-mon[65103]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:35:53.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:52 smithi150 ceph-mon[65103]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:35:53.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:52 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:35:53.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:52 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:53.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:52 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mgr 2024-04-22T21:35:53.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:52 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:53.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:52 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T21:35:53.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:52 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-22T21:35:53.256 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:35:53.257 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:35:53.257 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-22T21:35:53.257 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 1 2024-04-22T21:35:53.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:35:53.257 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:35:53.257 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:35:53.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:35:53.257 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:35:53.257 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:35:53.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:35:53.257 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": {}, 2024-04-22T21:35:53.257 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:35:53.257 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:35:53.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:35:53.258 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:35:53.258 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-22T21:35:53.258 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:35:53.258 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:35:53.258 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:35:53.609 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:35:53.609 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:35:53.609 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:35:53.610 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi130", 2024-04-22T21:35:53.610 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:35:53.610 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "0/2 daemons upgraded", 2024-04-22T21:35:53.610 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading mon daemons", 2024-04-22T21:35:53.610 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:35:53.610 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:35:53.923 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[21867]: Upgrade: It appears safe to stop mon.a 2024-04-22T21:35:53.923 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[21867]: from='client.15969 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:53.923 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[21867]: from='client.15975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:53.923 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[21867]: from='client.? 172.21.15.130:0/607284778' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:53.924 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[21867]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:53.924 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:35:53.924 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:35:53.924 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[21867]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:53.924 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[26394]: Upgrade: It appears safe to stop mon.a 2024-04-22T21:35:53.924 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[26394]: from='client.15969 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:53.924 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[26394]: from='client.15975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:53.924 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[26394]: from='client.? 172.21.15.130:0/607284778' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:53.924 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:53.924 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:35:53.924 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:35:53.924 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:53 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:54.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:53 smithi150 ceph-mon[65103]: Upgrade: It appears safe to stop mon.a 2024-04-22T21:35:54.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:53 smithi150 ceph-mon[65103]: from='client.15969 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:54.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:53 smithi150 ceph-mon[65103]: from='client.15975 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:54.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:53 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/607284778' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:35:54.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:53 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='mgr.y' 2024-04-22T21:35:54.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:53 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:35:54.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:53 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:35:54.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:53 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:35:54.935 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:54 smithi130 ceph-mon[21867]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:54.935 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:54 smithi130 ceph-mon[21867]: from='client.15981 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:54.935 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:54 smithi130 ceph-mon[21867]: Upgrade: Updating mon.a 2024-04-22T21:35:54.935 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:54 smithi130 ceph-mon[21867]: Deploying daemon mon.a on smithi130 2024-04-22T21:35:54.935 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:54 smithi130 ceph-mon[21867]: from='client.15993 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:54.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:54 smithi130 ceph-mon[26394]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:54.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:54 smithi130 ceph-mon[26394]: from='client.15981 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:54.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:54 smithi130 ceph-mon[26394]: Upgrade: Updating mon.a 2024-04-22T21:35:54.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:54 smithi130 ceph-mon[26394]: Deploying daemon mon.a on smithi130 2024-04-22T21:35:54.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:54 smithi130 ceph-mon[26394]: from='client.15993 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:55.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:54 smithi150 ceph-mon[65103]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:35:55.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:54 smithi150 ceph-mon[65103]: from='client.15981 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:55.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:54 smithi150 ceph-mon[65103]: Upgrade: Updating mon.a 2024-04-22T21:35:55.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:54 smithi150 ceph-mon[65103]: Deploying daemon mon.a on smithi130 2024-04-22T21:35:55.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:54 smithi150 ceph-mon[65103]: from='client.15993 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:35:55.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:55 smithi130 systemd[1]: Stopping Ceph mon.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:35:55.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a[21863]: 2024-04-22T21:35:55.507+0000 7f7865c7a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:35:55.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a[21863]: 2024-04-22T21:35:55.507+0000 7f7865c7a700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-22T21:35:55.924 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:55 smithi130 podman[82661]: 2024-04-22 21:35:55.681065891 +0000 UTC m=+0.279177395 container died e010afa4af4ddf25f78c352e5d94c39d6046b98fe1b32aa7d6a6e5138b5b993b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, vcs-type=git, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, maintainer=Guillaume Abrioux , architecture=x86_64, ceph=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, release=754, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True) 2024-04-22T21:35:56.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:56 smithi130 podman[82661]: 2024-04-22 21:35:56.04084196 +0000 UTC m=+0.638953449 container cleanup e010afa4af4ddf25f78c352e5d94c39d6046b98fe1b32aa7d6a6e5138b5b993b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, version=8, GIT_BRANCH=HEAD, ceph=True, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, name=centos-stream, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8) 2024-04-22T21:35:56.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:56 smithi130 bash[82661]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a 2024-04-22T21:35:56.648 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:56 smithi130 podman[82675]: 2024-04-22 21:35:56.341357279 +0000 UTC m=+0.664873662 container remove e010afa4af4ddf25f78c352e5d94c39d6046b98fe1b32aa7d6a6e5138b5b993b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.buildah.version=1.19.8, version=8, ceph=True, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, release=754) 2024-04-22T21:35:56.649 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:56 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-22T21:35:56.649 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:56 smithi130 systemd[1]: Stopped Ceph mon.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:35:56.649 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:56 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.a.service: Consumed 26.476s CPU time. 2024-04-22T21:35:57.035 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:56 smithi130 systemd[1]: Starting Ceph mon.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:35:57.035 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:56 smithi130 podman[82810]: 2024-04-22 21:35:56.789152417 +0000 UTC m=+0.020867100 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:35:57.035 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:56 smithi130 podman[82810]: 2024-04-22 21:35:56.910261964 +0000 UTC m=+0.141976618 container create 9db2f699adb7aae2a77ebd428c29aa03d2d7d53147a071e1a364d002397e76f8 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:35:57.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 podman[82810]: 2024-04-22 21:35:57.146348041 +0000 UTC m=+0.378062696 container init 9db2f699adb7aae2a77ebd428c29aa03d2d7d53147a071e1a364d002397e76f8 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-56e81df, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:35:57.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 podman[82810]: 2024-04-22 21:35:57.149380829 +0000 UTC m=+0.381095492 container start 9db2f699adb7aae2a77ebd428c29aa03d2d7d53147a071e1a364d002397e76f8 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-22T21:35:57.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T21:35:57.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev), process ceph-mon, pid 2 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: pidfile_write: ignore empty --pid-file 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: load: jerasure load: lrc 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: RocksDB version: 7.9.2 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Git sha 0 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Compile date 2024-04-22 15:21:46 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: DB SUMMARY 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: DB Session ID: WCPAAQON8SXDFO9A9I5J 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: CURRENT file: CURRENT 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: MANIFEST file: MANIFEST-000015 size: 3095 Bytes 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000063.sst 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000061.log size: 669601 ; 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.error_if_exists: 0 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.create_if_missing: 0 2024-04-22T21:35:57.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.paranoid_checks: 1 2024-04-22T21:35:57.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-22T21:35:57.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T21:35:57.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-22T21:35:57.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.env: 0x556fd9a45cc0 2024-04-22T21:35:57.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.fs: PosixFileSystem 2024-04-22T21:35:57.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.info_log: 0x556fdb47fbe0 2024-04-22T21:35:57.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T21:35:57.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.statistics: (nil) 2024-04-22T21:35:57.299 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.use_fsync: 0 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_log_file_size: 0 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.allow_fallocate: 1 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.use_direct_reads: 0 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.db_log_dir: 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.wal_dir: 2024-04-22T21:35:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.write_buffer_manager: 0x556fdb413220 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T21:35:57.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T21:35:57.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T21:35:57.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T21:35:57.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.unordered_write: 0 2024-04-22T21:35:57.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T21:35:57.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T21:35:57.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T21:35:57.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.row_cache: None 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.wal_filter: None 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.two_write_queues: 0 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.wal_compression: 0 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.atomic_flush: 0 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.log_readahead_size: 0 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T21:35:57.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_background_jobs: 2 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_background_compactions: -1 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_subcompactions: 1 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T21:35:57.304 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_open_files: -1 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_background_flushes: -1 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Compression algorithms supported: 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: kZSTD supported: 0 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: kXpressCompression supported: 0 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T21:35:57.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: kZlibCompression supported: 1 2024-04-22T21:35:57.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: kSnappyCompression supported: 1 2024-04-22T21:35:57.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: kLZ4Compression supported: 1 2024-04-22T21:35:57.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: kBZip2Compression supported: 0 2024-04-22T21:35:57.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T21:35:57.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-22T21:35:57.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-22T21:35:57.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-22T21:35:57.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T21:35:57.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.merge_operator: 2024-04-22T21:35:57.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compaction_filter: None 2024-04-22T21:35:57.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T21:35:57.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T21:35:57.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T21:35:57.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T21:35:57.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556fdb47fdc0) 2024-04-22T21:35:57.307 INFO:journalctl@ceph.mon.a.smithi130.stdout: cache_index_and_filter_blocks: 1 2024-04-22T21:35:57.307 INFO:journalctl@ceph.mon.a.smithi130.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T21:35:57.307 INFO:journalctl@ceph.mon.a.smithi130.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T21:35:57.307 INFO:journalctl@ceph.mon.a.smithi130.stdout: pin_top_level_index_and_filter: 1 2024-04-22T21:35:57.307 INFO:journalctl@ceph.mon.a.smithi130.stdout: index_type: 0 2024-04-22T21:35:57.307 INFO:journalctl@ceph.mon.a.smithi130.stdout: data_block_index_type: 0 2024-04-22T21:35:57.307 INFO:journalctl@ceph.mon.a.smithi130.stdout: index_shortening: 1 2024-04-22T21:35:57.307 INFO:journalctl@ceph.mon.a.smithi130.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T21:35:57.307 INFO:journalctl@ceph.mon.a.smithi130.stdout: checksum: 4 2024-04-22T21:35:57.307 INFO:journalctl@ceph.mon.a.smithi130.stdout: no_block_cache: 0 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache: 0x556fdb4451f0 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache_name: BinnedLRUCache 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache_options: 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: capacity : 536870912 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: num_shard_bits : 4 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: strict_capacity_limit : 0 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: high_pri_pool_ratio: 0.000 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache_compressed: (nil) 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: persistent_cache: (nil) 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_size: 4096 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_size_deviation: 10 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_restart_interval: 16 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: index_block_restart_interval: 1 2024-04-22T21:35:57.308 INFO:journalctl@ceph.mon.a.smithi130.stdout: metadata_block_size: 4096 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout: partition_filters: 0 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_delta_encoding: 1 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout: filter_policy: bloomfilter 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout: whole_key_filtering: 1 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout: verify_compression: 0 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout: read_amp_bytes_per_bit: 0 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout: format_version: 5 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout: enable_index_compression: 1 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_align: 0 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout: max_auto_readahead_size: 262144 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout: prepopulate_block_cache: 0 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout: initial_auto_readahead_size: 8192 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout: num_file_reads_for_auto_readahead: 2 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T21:35:57.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compression: NoCompression 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.num_levels: 7 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T21:35:57.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T21:35:57.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-22T21:35:57.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.arena_block_size: 1048576 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T21:35:57.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.inplace_update_support: 0 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.bloom_locality: 0 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.max_successive_merges: 0 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.ttl: 2592000 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T21:35:57.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.enable_blob_files: false 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.min_blob_size: 0 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.blob_file_starting_level: 0 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 63.sst 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T21:35:57.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 65, last_sequence is 29113, log_number is 61,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 61 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e1b579a8-aa90-4ad1-bc52-a43d3a590e44 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713821757179748, "job": 1, "event": "recovery_started", "wal_files": [61]} 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #61 mode 2 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713821757183052, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 66, "file_size": 654053, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 29114, "largest_seqno": 29925, "table_properties": {"data_size": 650390, "index_size": 1599, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1029, "raw_key_size": 10658, "raw_average_key_size": 26, "raw_value_size": 642382, "raw_average_value_size": 1609, "num_data_blocks": 70, "num_entries": 399, "num_filter_entries": 399, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1713821757, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e1b579a8-aa90-4ad1-bc52-a43d3a590e44", "db_session_id": "WCPAAQON8SXDFO9A9I5J", "orig_file_number": 66, "seqno_to_time_mapping": "N/A"}} 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713821757183115, "job": 1, "event": "recovery_finished"} 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [db/version_set.cc:5047] Creating manifest 68 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000061.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x556fdb55a000 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: DB pointer 0x556fdb544000 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** DB Stats ** 2024-04-22T21:35:57.316 INFO:journalctl@ceph.mon.a.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: L0 1/0 638.72 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 314.2 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: L6 1/0 9.96 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: Sum 2/0 10.59 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 314.2 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 314.2 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-22T21:35:57.317 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 314.2 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T21:35:57.318 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative compaction: 0.00 GB write, 115.32 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval compaction: 0.00 GB write, 115.32 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout: Block cache BinnedLRUCache@0x556fdb4451f0#2 capacity: 512.00 MB usage: 2.98 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.16 KB,0.000220537%) IndexBlock(1,1.83 KB,0.000348687%) Misc(1,0.00 KB,0%) 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: starting mon.a rank 0 at public addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] at bind addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: mon.a@-1(???) e3 preinit fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: mon.a@-1(???).mds e1 new map 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: mon.a@-1(???).mds e1 print_map 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout: e1 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout: legacy client fscid: -1 2024-04-22T21:35:57.319 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-04-22T21:35:57.320 INFO:journalctl@ceph.mon.a.smithi130.stdout: No filesystems configured 2024-04-22T21:35:57.320 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: mon.a@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-22T21:35:57.320 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T21:35:57.320 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T21:35:57.320 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T21:35:57.320 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 ceph-mon[82829]: mon.a@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-04-22T21:35:57.320 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 bash[82810]: 9db2f699adb7aae2a77ebd428c29aa03d2d7d53147a071e1a364d002397e76f8 2024-04-22T21:35:57.320 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:57 smithi130 systemd[1]: Started Ceph mon.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:35:58.972 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ignoring --setuser ceph since I am not root 2024-04-22T21:35:58.972 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ignoring --setgroup ceph since I am not root 2024-04-22T21:35:58.972 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mgr[60104]: -- 172.21.15.130:0/2257640313 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56081a3885a0 con 0x56081a35cc00 2024-04-22T21:35:58.972 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:35:58.804+0000 7fd763e53200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:35:58.972 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:35:58.989+0000 7fd763e53200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:35:58.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:58.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:58.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:35:58.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: mon.a calling monitor election 2024-04-22T21:35:58.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T21:35:58.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: monmap epoch 3 2024-04-22T21:35:58.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:35:58.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: last_changed 2024-04-22T21:06:22.254615+0000 2024-04-22T21:35:58.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: created 2024-04-22T21:04:51.246975+0000 2024-04-22T21:35:58.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: min_mon_release 17 (quincy) 2024-04-22T21:35:58.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: election_strategy: 1 2024-04-22T21:35:58.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-04-22T21:35:58.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: 1: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-04-22T21:35:58.974 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: 2: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.b 2024-04-22T21:35:58.974 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: fsmap 2024-04-22T21:35:58.974 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T21:35:58.974 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: mgrmap e43: y(active, since 4m), standbys: x 2024-04-22T21:35:58.974 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: overall HEALTH_OK 2024-04-22T21:35:58.974 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: from='mgr.25315 ' entity='' 2024-04-22T21:35:58.974 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[82829]: mgrmap e44: y(active, since 4m), standbys: x 2024-04-22T21:35:58.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:58.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:58.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:35:58.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: mon.a calling monitor election 2024-04-22T21:35:58.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: monmap epoch 3 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: last_changed 2024-04-22T21:06:22.254615+0000 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: created 2024-04-22T21:04:51.246975+0000 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: min_mon_release 17 (quincy) 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: election_strategy: 1 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: 1: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: 2: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.b 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: fsmap 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: mgrmap e43: y(active, since 4m), standbys: x 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: overall HEALTH_OK 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: from='mgr.25315 ' entity='' 2024-04-22T21:35:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:35:58 smithi130 ceph-mon[26394]: mgrmap e44: y(active, since 4m), standbys: x 2024-04-22T21:35:58.982 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: ignoring --setuser ceph since I am not root 2024-04-22T21:35:58.982 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: ignoring --setgroup ceph since I am not root 2024-04-22T21:35:58.982 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mgr[58012]: -- 172.21.15.150:0/2091312267 <== mon.1 v2:172.21.15.130:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d72f0dc5a0 con 0x55d72f0b0800 2024-04-22T21:35:58.982 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:35:58.814+0000 7f52c5102200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T21:35:58.982 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:35:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:35:58.999+0000 7f52c5102200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:35:58.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:58.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:35:58.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: from='mgr.25315 172.21.15.130:0/1731546491' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:35:58.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: mon.a calling monitor election 2024-04-22T21:35:58.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T21:35:58.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: monmap epoch 3 2024-04-22T21:35:58.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:35:58.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: last_changed 2024-04-22T21:06:22.254615+0000 2024-04-22T21:35:58.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: created 2024-04-22T21:04:51.246975+0000 2024-04-22T21:35:58.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: min_mon_release 17 (quincy) 2024-04-22T21:35:58.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: election_strategy: 1 2024-04-22T21:35:58.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-04-22T21:35:58.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: 1: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-04-22T21:35:58.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: 2: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.b 2024-04-22T21:35:58.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: fsmap 2024-04-22T21:35:58.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T21:35:58.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: mgrmap e43: y(active, since 4m), standbys: x 2024-04-22T21:35:58.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: overall HEALTH_OK 2024-04-22T21:35:58.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: from='mgr.25315 ' entity='' 2024-04-22T21:35:58.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:35:58 smithi150 ceph-mon[65103]: mgrmap e44: y(active, since 4m), standbys: x 2024-04-22T21:35:59.295 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:35:59.092+0000 7fd763e53200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:35:59.349 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:35:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:35:59.103+0000 7f52c5102200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T21:35:59.795 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:35:59.405+0000 7fd763e53200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:35:59.796 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:35:59 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:35:59.489+0000 7fd763e53200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:35:59.849 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:35:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:35:59.412+0000 7f52c5102200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T21:35:59.849 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:35:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:35:59.495+0000 7f52c5102200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T21:36:01.014 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:00 smithi130 ceph-mon[26394]: mgrmap e45: y(active, since 4m), standbys: x 2024-04-22T21:36:01.014 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:00 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:00.684+0000 7fd763e53200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:36:01.014 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:00 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:00.857+0000 7fd763e53200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:36:01.015 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:00 smithi130 ceph-mon[82829]: mgrmap e45: y(active, since 4m), standbys: x 2024-04-22T21:36:01.017 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:00 smithi150 ceph-mon[65103]: mgrmap e45: y(active, since 4m), standbys: x 2024-04-22T21:36:01.017 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:00 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:00.688+0000 7f52c5102200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T21:36:01.017 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:00 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:00.861+0000 7f52c5102200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T21:36:01.017 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:01 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:01.034+0000 7f52c5102200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:36:01.295 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:01 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:01.031+0000 7fd763e53200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T21:36:01.565 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:01 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:01.326+0000 7f52c5102200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:36:01.565 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:01 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:01.420+0000 7f52c5102200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:36:01.614 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:01 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:01.327+0000 7fd763e53200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T21:36:01.614 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:01 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:01.421+0000 7fd763e53200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T21:36:01.849 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:01 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:01.628+0000 7f52c5102200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:36:01.849 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:01 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:01.827+0000 7f52c5102200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:36:02.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:01 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:01.632+0000 7fd763e53200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T21:36:02.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:01 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:01.829+0000 7fd763e53200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T21:36:02.795 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:02.444+0000 7fd763e53200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:36:02.845 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:02.431+0000 7f52c5102200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T21:36:03.099 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:02.862+0000 7f52c5102200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:36:03.099 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:02.961+0000 7f52c5102200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:36:03.177 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:02.881+0000 7fd763e53200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T21:36:03.177 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:02.980+0000 7fd763e53200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T21:36:03.545 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:03.194+0000 7fd763e53200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T21:36:03.599 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:03.168+0000 7f52c5102200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T21:36:04.033 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:03.706+0000 7f52c5102200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:36:04.033 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:03.801+0000 7f52c5102200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:36:04.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:03.742+0000 7fd763e53200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T21:36:04.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:03.838+0000 7fd763e53200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T21:36:04.349 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:04.049+0000 7f52c5102200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:36:04.349 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:04.186+0000 7f52c5102200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:36:04.349 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:04.287+0000 7f52c5102200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:36:04.438 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:04 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:04.091+0000 7fd763e53200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T21:36:04.439 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:04 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:04.238+0000 7fd763e53200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T21:36:04.439 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:04 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:04.337+0000 7fd763e53200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T21:36:04.439 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:04 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:04.456+0000 7fd763e53200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:36:04.695 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:04.406+0000 7f52c5102200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T21:36:04.695 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:04.713+0000 7f52c5102200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:36:05.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:04 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:04.767+0000 7fd763e53200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T21:36:05.593 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:05.324+0000 7f52c5102200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:36:05.593 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:05.548+0000 7f52c5102200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:36:05.593 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: [22/Apr/2024:21:36:05] ENGINE Bus STARTING 2024-04-22T21:36:05.695 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:05.390+0000 7fd763e53200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T21:36:05.695 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:05.624+0000 7fd763e53200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T21:36:05.695 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-mon[26394]: Standby manager daemon x restarted 2024-04-22T21:36:05.695 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-mon[26394]: Standby manager daemon x started 2024-04-22T21:36:05.695 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/2566954248' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:36:05.695 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/2566954248' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:36:05.695 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/2566954248' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:36:05.695 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-mon[26394]: from='mgr.? 172.21.15.150:0/2566954248' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:36:05.696 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-mon[82829]: Standby manager daemon x restarted 2024-04-22T21:36:05.696 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-mon[82829]: Standby manager daemon x started 2024-04-22T21:36:05.696 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-mon[82829]: from='mgr.? 172.21.15.150:0/2566954248' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:36:05.696 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-mon[82829]: from='mgr.? 172.21.15.150:0/2566954248' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:36:05.696 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-mon[82829]: from='mgr.? 172.21.15.150:0/2566954248' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:36:05.696 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-mon[82829]: from='mgr.? 172.21.15.150:0/2566954248' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:36:05.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-mon[65103]: Standby manager daemon x restarted 2024-04-22T21:36:05.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-mon[65103]: Standby manager daemon x started 2024-04-22T21:36:05.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-mon[65103]: from='mgr.? 172.21.15.150:0/2566954248' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T21:36:05.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-mon[65103]: from='mgr.? 172.21.15.150:0/2566954248' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T21:36:05.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-mon[65103]: from='mgr.? 172.21.15.150:0/2566954248' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T21:36:05.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-mon[65103]: from='mgr.? 172.21.15.150:0/2566954248' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T21:36:05.850 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: CherryPy Checker: 2024-04-22T21:36:05.850 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: The Application mounted at '' has an empty config. 2024-04-22T21:36:05.850 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: 2024-04-22T21:36:05.850 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: [22/Apr/2024:21:36:05] ENGINE Serving on http://:::9283 2024-04-22T21:36:05.850 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:36:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x[58008]: [22/Apr/2024:21:36:05] ENGINE Bus STARTED 2024-04-22T21:36:06.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:36:05] ENGINE Bus STARTING 2024-04-22T21:36:06.046 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CherryPy Checker: 2024-04-22T21:36:06.046 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: The Application mounted at '' has an empty config. 2024-04-22T21:36:06.046 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:06.046 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:36:05] ENGINE Serving on http://:::9283 2024-04-22T21:36:06.046 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:05 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:36:05] ENGINE Bus STARTED 2024-04-22T21:36:06.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: mgrmap e46: y(active, since 4m), standbys: x 2024-04-22T21:36:06.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: Active manager daemon y restarted 2024-04-22T21:36:06.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: Activating manager daemon y 2024-04-22T21:36:06.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: osdmap e113: 8 total, 8 up, 8 in 2024-04-22T21:36:06.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: mgrmap e47: y(active, starting, since 0.0111185s), standbys: x 2024-04-22T21:36:06.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:36:06.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:36:06.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:36:06.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: Manager daemon y is now available 2024-04-22T21:36:06.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:36:06.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:36:06.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:36:06.966 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:06.643+0000 7fd7212f0700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-22T21:36:06.967 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: mgrmap e46: y(active, since 4m), standbys: x 2024-04-22T21:36:06.967 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: Active manager daemon y restarted 2024-04-22T21:36:06.967 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: Activating manager daemon y 2024-04-22T21:36:06.967 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: osdmap e113: 8 total, 8 up, 8 in 2024-04-22T21:36:06.967 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: mgrmap e47: y(active, starting, since 0.0111185s), standbys: x 2024-04-22T21:36:06.967 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:36:06.967 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:36:06.967 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:36:06.967 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:36:06.967 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:36:06.967 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:36:06.967 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:36:06.967 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: Manager daemon y is now available 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: mgrmap e46: y(active, since 4m), standbys: x 2024-04-22T21:36:06.968 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: Active manager daemon y restarted 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: Activating manager daemon y 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: osdmap e113: 8 total, 8 up, 8 in 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: mgrmap e47: y(active, starting, since 0.0111185s), standbys: x 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:36:06.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:36:06.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T21:36:06.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T21:36:06.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T21:36:06.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: Manager daemon y is now available 2024-04-22T21:36:06.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:36:06.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T21:36:06.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:06 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T21:36:07.888 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:07 smithi150 ceph-mon[65103]: [22/Apr/2024:21:36:06] ENGINE Bus STARTING 2024-04-22T21:36:07.888 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:07 smithi150 ceph-mon[65103]: [22/Apr/2024:21:36:06] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:36:07.889 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:07 smithi150 ceph-mon[65103]: [22/Apr/2024:21:36:06] ENGINE Serving on http://172.21.15.130:8765 2024-04-22T21:36:07.889 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:07 smithi150 ceph-mon[65103]: [22/Apr/2024:21:36:06] ENGINE Bus STARTED 2024-04-22T21:36:07.889 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:07 smithi150 ceph-mon[65103]: mgrmap e48: y(active, since 1.01045s), standbys: x 2024-04-22T21:36:07.889 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:07 smithi150 ceph-mon[65103]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:36:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:07 smithi130 ceph-mon[26394]: [22/Apr/2024:21:36:06] ENGINE Bus STARTING 2024-04-22T21:36:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:07 smithi130 ceph-mon[26394]: [22/Apr/2024:21:36:06] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:36:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:07 smithi130 ceph-mon[26394]: [22/Apr/2024:21:36:06] ENGINE Serving on http://172.21.15.130:8765 2024-04-22T21:36:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:07 smithi130 ceph-mon[26394]: [22/Apr/2024:21:36:06] ENGINE Bus STARTED 2024-04-22T21:36:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:07 smithi130 ceph-mon[26394]: mgrmap e48: y(active, since 1.01045s), standbys: x 2024-04-22T21:36:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:07 smithi130 ceph-mon[26394]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:36:08.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:07 smithi130 ceph-mon[82829]: [22/Apr/2024:21:36:06] ENGINE Bus STARTING 2024-04-22T21:36:08.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:07 smithi130 ceph-mon[82829]: [22/Apr/2024:21:36:06] ENGINE Serving on https://172.21.15.130:7150 2024-04-22T21:36:08.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:07 smithi130 ceph-mon[82829]: [22/Apr/2024:21:36:06] ENGINE Serving on http://172.21.15.130:8765 2024-04-22T21:36:08.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:07 smithi130 ceph-mon[82829]: [22/Apr/2024:21:36:06] ENGINE Bus STARTED 2024-04-22T21:36:08.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:07 smithi130 ceph-mon[82829]: mgrmap e48: y(active, since 1.01045s), standbys: x 2024-04-22T21:36:08.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:07 smithi130 ceph-mon[82829]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:36:08.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:08 smithi150 ceph-mon[65103]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:36:08.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:08 smithi150 ceph-mon[65103]: mgrmap e49: y(active, since 2s), standbys: x 2024-04-22T21:36:08.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:08 smithi130 ceph-mon[82829]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:36:08.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:08 smithi130 ceph-mon[82829]: mgrmap e49: y(active, since 2s), standbys: x 2024-04-22T21:36:08.899 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:08 smithi130 ceph-mon[26394]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:36:08.899 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:08 smithi130 ceph-mon[26394]: mgrmap e49: y(active, since 2s), standbys: x 2024-04-22T21:36:09.181 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:36:08] "GET /metrics HTTP/1.1" 200 35136 "" "Prometheus/2.43.0" 2024-04-22T21:36:09.888 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:09 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:10.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:09 smithi130 ceph-mon[26394]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:10.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:09 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:11.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:10 smithi130 ceph-mon[82829]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:36:11.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:10 smithi130 ceph-mon[82829]: mgrmap e50: y(active, since 4s), standbys: x 2024-04-22T21:36:11.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:10 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:11.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:10 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:11.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:10 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:11.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:10 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:11.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:10 smithi130 ceph-mon[26394]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:36:11.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:10 smithi130 ceph-mon[26394]: mgrmap e50: y(active, since 4s), standbys: x 2024-04-22T21:36:11.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:10 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:11.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:10 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:11.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:10 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:11.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:10 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:11.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:10 smithi150 ceph-mon[65103]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:36:11.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:10 smithi150 ceph-mon[65103]: mgrmap e50: y(active, since 4s), standbys: x 2024-04-22T21:36:11.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:10 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:11.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:10 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:11.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:10 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:11.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:10 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:12.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:12 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:12.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:12 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:12.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:12 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:12.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:12 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:12.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:12 smithi150 ceph-mon[65103]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:36:12.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:12 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:12.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:12 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:12.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:12 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:12.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:12 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:12.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:12 smithi130 ceph-mon[26394]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:36:12.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:12 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:12.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:12 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:12.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:12 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:12.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:12 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:12.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:12 smithi130 ceph-mon[82829]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:36:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:13 smithi130 ceph-mon[26394]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:36:14.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:13 smithi130 ceph-mon[82829]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:36:14.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:13 smithi150 ceph-mon[65103]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T21:36:16.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:16.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:16.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:16.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:36:16.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:36:16.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:36:16.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[26394]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:36:16.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[26394]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:36:16.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[26394]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:36:16.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[26394]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:36:16.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[26394]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:36:16.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:16.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:16.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:16.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:36:16.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[82829]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:36:16.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[82829]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:36:16.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[82829]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:36:16.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[82829]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:36:16.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[82829]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:36:16.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[82829]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:36:16.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:15 smithi130 ceph-mon[82829]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:36:16.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:15 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:16.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:15 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:16.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:15 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:16.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:15 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:36:16.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:15 smithi150 ceph-mon[65103]: Updating smithi130:/etc/ceph/ceph.conf 2024-04-22T21:36:16.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:15 smithi150 ceph-mon[65103]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-22T21:36:16.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:15 smithi150 ceph-mon[65103]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:36:16.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:15 smithi150 ceph-mon[65103]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T21:36:16.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:15 smithi150 ceph-mon[65103]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:36:16.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:15 smithi150 ceph-mon[65103]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-04-22T21:36:16.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:15 smithi150 ceph-mon[65103]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T21:36:17.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[26394]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:36:17.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[26394]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:36:17.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:36:17.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:17.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T21:36:17.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-22T21:36:17.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[26394]: Upgrade: It appears safe to stop mon.c 2024-04-22T21:36:17.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[82829]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:36:17.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[82829]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:36:17.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:36:17.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:17.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T21:36:17.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-22T21:36:17.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:16 smithi130 ceph-mon[82829]: Upgrade: It appears safe to stop mon.c 2024-04-22T21:36:17.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:16 smithi150 ceph-mon[65103]: Updating smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:36:17.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:16 smithi150 ceph-mon[65103]: Updating smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T21:36:17.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:16 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:16 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:16 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:16 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:16 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:17.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:16 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:36:17.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:16 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:17.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:16 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T21:36:17.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:16 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-22T21:36:17.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:16 smithi150 ceph-mon[65103]: Upgrade: It appears safe to stop mon.c 2024-04-22T21:36:18.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[82829]: Upgrade: Updating mon.c 2024-04-22T21:36:18.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:18.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:36:18.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:36:18.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:18.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[82829]: Deploying daemon mon.c on smithi130 2024-04-22T21:36:18.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[82829]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:36:18.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[26394]: Upgrade: Updating mon.c 2024-04-22T21:36:18.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:18.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:36:18.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:36:18.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[26394]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:18.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[26394]: Deploying daemon mon.c on smithi130 2024-04-22T21:36:18.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-mon[26394]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:36:18.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:18 smithi150 ceph-mon[65103]: Upgrade: Updating mon.c 2024-04-22T21:36:18.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:18 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:18.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:18 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:36:18.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:18 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:36:18.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:18 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:18.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:18 smithi150 ceph-mon[65103]: Deploying daemon mon.c on smithi130 2024-04-22T21:36:18.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:18 smithi150 ceph-mon[65103]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:36:19.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:36:18] "GET /metrics HTTP/1.1" 200 35136 "" "Prometheus/2.43.0" 2024-04-22T21:36:19.627 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:19 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:19.628 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:19 smithi130 systemd[1]: Stopping Ceph mon.c for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:36:19.628 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:19 smithi130 ceph-mon[26394]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:19.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:19 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:20.027 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:19 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c[26390]: 2024-04-22T21:36:19.644+0000 7fa47e931700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:36:20.027 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:19 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c[26390]: 2024-04-22T21:36:19.644+0000 7fa47e931700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-04-22T21:36:20.027 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:19 smithi130 podman[85037]: 2024-04-22 21:36:19.747132136 +0000 UTC m=+0.228235615 container died 0e0f9de395abf1723872900f43ff8e152497183a38ba475235f4b4a1370c33c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c, io.k8s.display-name=CentOS Stream 8, name=centos-stream, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, release=754) 2024-04-22T21:36:20.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:20 smithi130 podman[85037]: 2024-04-22 21:36:20.136174704 +0000 UTC m=+0.617278189 container cleanup 0e0f9de395abf1723872900f43ff8e152497183a38ba475235f4b4a1370c33c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, com.redhat.component=centos-stream-container, ceph=True, version=8, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-22T21:36:20.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:20 smithi130 bash[85037]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c 2024-04-22T21:36:20.736 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:20 smithi130 podman[85052]: 2024-04-22 21:36:20.395937846 +0000 UTC m=+0.640819753 container remove 0e0f9de395abf1723872900f43ff8e152497183a38ba475235f4b4a1370c33c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, release=754, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.openshift.expose-services=, name=centos-stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, version=8, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-22T21:36:20.736 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:20 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-22T21:36:20.736 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:20 smithi130 systemd[1]: Stopped Ceph mon.c for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:36:20.736 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:20 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.c.service: Consumed 16.773s CPU time. 2024-04-22T21:36:20.995 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:20 smithi130 systemd[1]: Starting Ceph mon.c for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:36:20.995 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:20 smithi130 podman[85186]: 2024-04-22 21:36:20.873472739 +0000 UTC m=+0.018091186 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:36:20.995 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 podman[85186]: 2024-04-22 21:36:21.013875756 +0000 UTC m=+0.158494193 container create cfb56e816511afdcf5ef78d1abd8665fb06c5ea2a9264ef5fe1107a2a66750ba (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-56e81df, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, ceph=True, CEPH_POINT_RELEASE=) 2024-04-22T21:36:21.273 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 podman[85186]: 2024-04-22 21:36:21.208131417 +0000 UTC m=+0.352749852 container init cfb56e816511afdcf5ef78d1abd8665fb06c5ea2a9264ef5fe1107a2a66750ba (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240416) 2024-04-22T21:36:21.273 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 podman[85186]: 2024-04-22 21:36:21.211036553 +0000 UTC m=+0.355654988 container start cfb56e816511afdcf5ef78d1abd8665fb06c5ea2a9264ef5fe1107a2a66750ba (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-56e81df, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True) 2024-04-22T21:36:21.273 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T21:36:21.273 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev), process ceph-mon, pid 2 2024-04-22T21:36:21.274 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: pidfile_write: ignore empty --pid-file 2024-04-22T21:36:21.274 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: load: jerasure load: lrc 2024-04-22T21:36:21.274 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: RocksDB version: 7.9.2 2024-04-22T21:36:21.274 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Git sha 0 2024-04-22T21:36:21.274 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Compile date 2024-04-22 15:21:46 2024-04-22T21:36:21.274 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: DB SUMMARY 2024-04-22T21:36:21.274 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: DB Session ID: IUYY2DP7TWKBAENVKMHJ 2024-04-22T21:36:21.274 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: CURRENT file: CURRENT 2024-04-22T21:36:21.274 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T21:36:21.274 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3210 Bytes 2024-04-22T21:36:21.274 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000060.sst 2024-04-22T21:36:21.274 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000058.log size: 2602922 ; 2024-04-22T21:36:21.275 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.error_if_exists: 0 2024-04-22T21:36:21.275 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.create_if_missing: 0 2024-04-22T21:36:21.275 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.paranoid_checks: 1 2024-04-22T21:36:21.275 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-22T21:36:21.275 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T21:36:21.275 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-22T21:36:21.275 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.env: 0x5633817dbcc0 2024-04-22T21:36:21.275 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.fs: PosixFileSystem 2024-04-22T21:36:21.275 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.info_log: 0x5633834cfbe0 2024-04-22T21:36:21.275 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T21:36:21.275 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.statistics: (nil) 2024-04-22T21:36:21.276 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.use_fsync: 0 2024-04-22T21:36:21.276 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_log_file_size: 0 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.allow_fallocate: 1 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.use_direct_reads: 0 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.db_log_dir: 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.wal_dir: 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T21:36:21.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.write_buffer_manager: 0x563383463220 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T21:36:21.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.unordered_write: 0 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.row_cache: None 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.wal_filter: None 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.two_write_queues: 0 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.wal_compression: 0 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.atomic_flush: 0 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.log_readahead_size: 0 2024-04-22T21:36:21.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_background_jobs: 2 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_background_compactions: -1 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_subcompactions: 1 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T21:36:21.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_open_files: -1 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_background_flushes: -1 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Compression algorithms supported: 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: kZSTD supported: 0 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: kXpressCompression supported: 0 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: kZlibCompression supported: 1 2024-04-22T21:36:21.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: kSnappyCompression supported: 1 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: kLZ4Compression supported: 1 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: kBZip2Compression supported: 0 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.merge_operator: 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compaction_filter: None 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T21:36:21.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5633834cfdc0) 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: cache_index_and_filter_blocks: 1 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: pin_top_level_index_and_filter: 1 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_type: 0 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: data_block_index_type: 0 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_shortening: 1 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: checksum: 4 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: no_block_cache: 0 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache: 0x5633834951f0 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_name: BinnedLRUCache 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_options: 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: capacity : 536870912 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: num_shard_bits : 4 2024-04-22T21:36:21.283 INFO:journalctl@ceph.mon.c.smithi130.stdout: strict_capacity_limit : 0 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: high_pri_pool_ratio: 0.000 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_compressed: (nil) 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: persistent_cache: (nil) 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_size: 4096 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_size_deviation: 10 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_restart_interval: 16 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_block_restart_interval: 1 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: metadata_block_size: 4096 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: partition_filters: 0 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_delta_encoding: 1 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: filter_policy: bloomfilter 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: whole_key_filtering: 1 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: verify_compression: 0 2024-04-22T21:36:21.284 INFO:journalctl@ceph.mon.c.smithi130.stdout: read_amp_bytes_per_bit: 0 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout: format_version: 5 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout: enable_index_compression: 1 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_align: 0 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout: max_auto_readahead_size: 262144 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout: prepopulate_block_cache: 0 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout: initial_auto_readahead_size: 8192 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout: num_file_reads_for_auto_readahead: 2 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compression: NoCompression 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.num_levels: 7 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T21:36:21.285 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T21:36:21.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T21:36:21.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.arena_block_size: 1048576 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T21:36:21.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.inplace_update_support: 0 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.bloom_locality: 0 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.max_successive_merges: 0 2024-04-22T21:36:21.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.ttl: 2592000 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.enable_blob_files: false 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.min_blob_size: 0 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T21:36:21.290 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.blob_file_starting_level: 0 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 62, last_sequence is 32929, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 52a13a85-bf98-465b-bbf3-ff135a81440d 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713821781240826, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713821781247246, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 1667262, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 32935, "largest_seqno": 33333, "table_properties": {"data_size": 1664327, "index_size": 1383, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 4627, "raw_average_key_size": 24, "raw_value_size": 1659893, "raw_average_value_size": 8645, "num_data_blocks": 62, "num_entries": 192, "num_filter_entries": 192, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1713821781, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "52a13a85-bf98-465b-bbf3-ff135a81440d", "db_session_id": "IUYY2DP7TWKBAENVKMHJ", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713821781247363, "job": 1, "event": "recovery_finished"} 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T21:36:21.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5633835aa000 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: DB pointer 0x563383594000 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** DB Stats ** 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-22T21:36:21.292 INFO:journalctl@ceph.mon.c.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: L0 1/0 1.59 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 560.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: L6 1/0 10.98 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: Sum 2/0 12.57 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 560.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 560.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 560.5 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T21:36:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative compaction: 0.00 GB write, 147.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval compaction: 0.00 GB write, 147.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout: Block cache BinnedLRUCache@0x5633834951f0#2 capacity: 512.00 MB usage: 2.05 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: starting mon.c rank 1 at public addrs [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] at bind addrs [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: mon.c@-1(???) e3 preinit fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:36:21.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: mon.c@-1(???).mds e1 new map 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: mon.c@-1(???).mds e1 print_map 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout: e1 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout: legacy client fscid: -1 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout: No filesystems configured 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: mon.c@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 ceph-mon[85205]: mon.c@-1(???).paxosservice(auth 1..28) refresh upgraded, format 0 -> 3 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 bash[85186]: cfb56e816511afdcf5ef78d1abd8665fb06c5ea2a9264ef5fe1107a2a66750ba 2024-04-22T21:36:21.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:21 smithi130 systemd[1]: Started Ceph mon.c for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:36:23.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: overall HEALTH_OK 2024-04-22T21:36:23.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: mon.c calling monitor election 2024-04-22T21:36:23.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: mon.b calling monitor election 2024-04-22T21:36:23.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:36:23.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:36:23.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:36:23.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: mon.a calling monitor election 2024-04-22T21:36:23.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T21:36:23.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: monmap epoch 4 2024-04-22T21:36:23.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: last_changed 2024-04-22T21:36:21.979063+0000 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: created 2024-04-22T21:04:51.246975+0000 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: min_mon_release 19 (squid) 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: election_strategy: 1 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: 1: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: 2: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.b 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: fsmap 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: osdmap e113: 8 total, 8 up, 8 in 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: mgrmap e50: y(active, since 16s), standbys: x 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: overall HEALTH_OK 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:23.245 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: overall HEALTH_OK 2024-04-22T21:36:23.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: mon.c calling monitor election 2024-04-22T21:36:23.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: mon.b calling monitor election 2024-04-22T21:36:23.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:36:23.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:36:23.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:36:23.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: mon.a calling monitor election 2024-04-22T21:36:23.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T21:36:23.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: monmap epoch 4 2024-04-22T21:36:23.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:36:23.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: last_changed 2024-04-22T21:36:21.979063+0000 2024-04-22T21:36:23.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: created 2024-04-22T21:04:51.246975+0000 2024-04-22T21:36:23.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: min_mon_release 19 (squid) 2024-04-22T21:36:23.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: election_strategy: 1 2024-04-22T21:36:23.247 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-04-22T21:36:23.247 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: 1: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-04-22T21:36:23.247 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: 2: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.b 2024-04-22T21:36:23.247 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: fsmap 2024-04-22T21:36:23.247 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: osdmap e113: 8 total, 8 up, 8 in 2024-04-22T21:36:23.247 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: mgrmap e50: y(active, since 16s), standbys: x 2024-04-22T21:36:23.247 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: overall HEALTH_OK 2024-04-22T21:36:23.247 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:23.247 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:22 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:23.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: overall HEALTH_OK 2024-04-22T21:36:23.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: mon.c calling monitor election 2024-04-22T21:36:23.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: mon.b calling monitor election 2024-04-22T21:36:23.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T21:36:23.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T21:36:23.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T21:36:23.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: mon.a calling monitor election 2024-04-22T21:36:23.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T21:36:23.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: monmap epoch 4 2024-04-22T21:36:23.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 2024-04-22T21:36:23.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: last_changed 2024-04-22T21:36:21.979063+0000 2024-04-22T21:36:23.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: created 2024-04-22T21:04:51.246975+0000 2024-04-22T21:36:23.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: min_mon_release 19 (squid) 2024-04-22T21:36:23.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: election_strategy: 1 2024-04-22T21:36:23.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-04-22T21:36:23.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: 1: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-04-22T21:36:23.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: 2: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.b 2024-04-22T21:36:23.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: fsmap 2024-04-22T21:36:23.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: osdmap e113: 8 total, 8 up, 8 in 2024-04-22T21:36:23.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: mgrmap e50: y(active, since 16s), standbys: x 2024-04-22T21:36:23.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: overall HEALTH_OK 2024-04-22T21:36:23.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:23.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:22 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:23.968 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:36:24.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:23 smithi130 ceph-mon[82829]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:36:24.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:23 smithi130 ceph-mon[85205]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:36:24.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:23 smithi150 ceph-mon[65103]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T21:36:24.647 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:36:24.647 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (2m) 14s ago 25m 28.6M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:36:24.647 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (2m) 14s ago 24m 83.7M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (2m) 14s ago 24m 51.6M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (4m) 14s ago 29m 448M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (13m) 14s ago 31m 475M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (27s) 14s ago 31m 33.4M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (77s) 14s ago 30m 38.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 starting - - - 2048M 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (3m) 14s ago 25m 16.1M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (3m) 14s ago 25m 15.8M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (29m) 14s ago 29m 54.9M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (29m) 14s ago 29m 57.4M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (28m) 14s ago 28m 50.6M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (28m) 14s ago 28m 53.0M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (27m) 14s ago 27m 55.5M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (27m) 14s ago 27m 52.8M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:36:24.648 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (27m) 14s ago 27m 54.5M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:36:24.649 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (26m) 14s ago 26m 53.8M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:36:24.649 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (3m) 14s ago 25m 69.5M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:36:24.649 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (24m) 14s ago 24m 79.0M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:36:24.649 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (24m) 14s ago 24m 76.2M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:36:25.068 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:36:25.068 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:36:25.068 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:36:25.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:36:25.068 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:36:25.068 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:36:25.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:36:25.068 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:36:25.068 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:36:25.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:36:25.068 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:36:25.068 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:36:25.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:36:25.069 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:36:25.069 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-22T21:36:25.069 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 5 2024-04-22T21:36:25.069 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:36:25.069 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:36:25.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:24 smithi150 ceph-mon[65103]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:25.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:24 smithi150 ceph-mon[65103]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:25.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:24 smithi150 ceph-mon[65103]: from='client.54150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:25.229 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:24 smithi130 ceph-mon[85205]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:25.229 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:24 smithi130 ceph-mon[85205]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:25.229 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:24 smithi130 ceph-mon[85205]: from='client.54150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:25.230 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:24 smithi130 ceph-mon[82829]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:25.230 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:24 smithi130 ceph-mon[82829]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:25.230 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:24 smithi130 ceph-mon[82829]: from='client.54150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:25.413 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:36:25.413 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:36:25.413 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:36:25.413 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi130", 2024-04-22T21:36:25.413 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:36:25.413 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "1/2 daemons upgraded", 2024-04-22T21:36:25.413 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading mon daemons", 2024-04-22T21:36:25.413 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:36:25.413 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:36:26.240 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:25 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/974082682' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:26.240 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:25 smithi130 ceph-mon[82829]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:26.240 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:25 smithi130 ceph-mon[82829]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:26.240 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:25 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/974082682' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:26.241 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:25 smithi130 ceph-mon[85205]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:26.241 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:25 smithi130 ceph-mon[85205]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:26.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:25 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/974082682' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:26.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:25 smithi150 ceph-mon[65103]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:26.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:25 smithi150 ceph-mon[65103]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:26.545 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:26 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:36:26.256+0000 7fd7212f0700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-22T21:36:27.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:27 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:27.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:27 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:27.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:27 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:27.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:27 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:27.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:27 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:27.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:27 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:28.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:28 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:28.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:28 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:28.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:28 smithi130 ceph-mon[85205]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:28.776 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:28 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:28.776 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:28 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:28.776 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:28 smithi130 ceph-mon[82829]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:28.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:28 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:28.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:28 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:28.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:28 smithi150 ceph-mon[65103]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:29.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:36:28] "GET /metrics HTTP/1.1" 200 37951 "" "Prometheus/2.43.0" 2024-04-22T21:36:29.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:29 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:29.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:29 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:29.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:29 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:30.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:30 smithi130 ceph-mon[82829]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:30.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:30 smithi130 ceph-mon[85205]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:30.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:30 smithi150 ceph-mon[65103]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:32.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[82829]: Detected new or changed devices on smithi130 2024-04-22T21:36:32.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:32.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:32.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:32.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:36:32.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:32.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[82829]: Reconfiguring mon.a (monmap changed)... 2024-04-22T21:36:32.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:36:32.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:36:32.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:32.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[82829]: Reconfiguring daemon mon.a on smithi130 2024-04-22T21:36:32.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[82829]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:32.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[85205]: Detected new or changed devices on smithi130 2024-04-22T21:36:32.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:32.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:32.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:32.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:36:32.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:32.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[85205]: Reconfiguring mon.a (monmap changed)... 2024-04-22T21:36:32.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:36:32.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:36:32.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:32.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[85205]: Reconfiguring daemon mon.a on smithi130 2024-04-22T21:36:32.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:32 smithi130 ceph-mon[85205]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:32.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:32 smithi150 ceph-mon[65103]: Detected new or changed devices on smithi130 2024-04-22T21:36:32.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:32.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:32.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:32.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:36:32.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:32.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:32 smithi150 ceph-mon[65103]: Reconfiguring mon.a (monmap changed)... 2024-04-22T21:36:32.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:36:32.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:36:32.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:32.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:32 smithi150 ceph-mon[65103]: Reconfiguring daemon mon.a on smithi130 2024-04-22T21:36:32.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:32 smithi150 ceph-mon[65103]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:34.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:34.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:34.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[82829]: Reconfiguring mgr.y (monmap changed)... 2024-04-22T21:36:34.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:36:34.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:36:34.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:34.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[82829]: Reconfiguring daemon mgr.y on smithi130 2024-04-22T21:36:34.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[82829]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:34.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:34.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:34.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[85205]: Reconfiguring mgr.y (monmap changed)... 2024-04-22T21:36:34.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:36:34.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:36:34.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:34.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[85205]: Reconfiguring daemon mgr.y on smithi130 2024-04-22T21:36:34.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:33 smithi130 ceph-mon[85205]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:34.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:33 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:34.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:33 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:34.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:33 smithi150 ceph-mon[65103]: Reconfiguring mgr.y (monmap changed)... 2024-04-22T21:36:34.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:33 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:36:34.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:33 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:36:34.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:33 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:34.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:33 smithi150 ceph-mon[65103]: Reconfiguring daemon mgr.y on smithi130 2024-04-22T21:36:34.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:33 smithi150 ceph-mon[65103]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:35.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:35.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:35.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[82829]: Reconfiguring mon.c (monmap changed)... 2024-04-22T21:36:35.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:36:35.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:36:35.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:35.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[82829]: Reconfiguring daemon mon.c on smithi130 2024-04-22T21:36:35.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:35.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:35.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[85205]: Reconfiguring mon.c (monmap changed)... 2024-04-22T21:36:35.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:36:35.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:36:35.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:35.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:35 smithi130 ceph-mon[85205]: Reconfiguring daemon mon.c on smithi130 2024-04-22T21:36:35.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:35.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:35.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:35 smithi150 ceph-mon[65103]: Reconfiguring mon.c (monmap changed)... 2024-04-22T21:36:35.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:36:35.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:36:35.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:35.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:35 smithi150 ceph-mon[65103]: Reconfiguring daemon mon.c on smithi130 2024-04-22T21:36:36.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:36 smithi130 ceph-mon[85205]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:36.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:36:36.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:36.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:36.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T21:36:36.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:36.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:36 smithi130 ceph-mon[82829]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:36.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:36:36.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:36.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:36.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T21:36:36.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:36.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:36 smithi150 ceph-mon[65103]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:36.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:36:36.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:36.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:36.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T21:36:36.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:37.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:37 smithi130 ceph-mon[82829]: Reconfiguring osd.0 (monmap changed)... 2024-04-22T21:36:37.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:37 smithi130 ceph-mon[82829]: Reconfiguring daemon osd.0 on smithi130 2024-04-22T21:36:37.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:37 smithi130 ceph-mon[85205]: Reconfiguring osd.0 (monmap changed)... 2024-04-22T21:36:37.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:37 smithi130 ceph-mon[85205]: Reconfiguring daemon osd.0 on smithi130 2024-04-22T21:36:37.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:37 smithi150 ceph-mon[65103]: Reconfiguring osd.0 (monmap changed)... 2024-04-22T21:36:37.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:37 smithi150 ceph-mon[65103]: Reconfiguring daemon osd.0 on smithi130 2024-04-22T21:36:38.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:38 smithi130 ceph-mon[85205]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:38.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:38 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:38.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:38 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:38.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:38 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T21:36:38.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:38 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:38.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:38 smithi130 ceph-mon[82829]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:38.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:38 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:38.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:38 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:38.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:38 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T21:36:38.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:38 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:38.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:38 smithi150 ceph-mon[65103]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:38.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:38 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:38.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:38 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:38.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:38 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T21:36:38.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:38 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:39.295 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:36:38] "GET /metrics HTTP/1.1" 200 37953 "" "Prometheus/2.43.0" 2024-04-22T21:36:39.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:39 smithi130 ceph-mon[85205]: Reconfiguring osd.1 (monmap changed)... 2024-04-22T21:36:39.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:39 smithi130 ceph-mon[85205]: Reconfiguring daemon osd.1 on smithi130 2024-04-22T21:36:39.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:39 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:39.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:39 smithi130 ceph-mon[82829]: Reconfiguring osd.1 (monmap changed)... 2024-04-22T21:36:39.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:39 smithi130 ceph-mon[82829]: Reconfiguring daemon osd.1 on smithi130 2024-04-22T21:36:39.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:39 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:39.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:39 smithi150 ceph-mon[65103]: Reconfiguring osd.1 (monmap changed)... 2024-04-22T21:36:39.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:39 smithi150 ceph-mon[65103]: Reconfiguring daemon osd.1 on smithi130 2024-04-22T21:36:39.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:39 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:41.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:40 smithi130 ceph-mon[85205]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:41.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:40 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:41.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:40 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:41.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:40 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T21:36:41.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:40 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:41.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:40 smithi130 ceph-mon[82829]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:41.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:40 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:41.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:40 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:41.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:40 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T21:36:41.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:40 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:41.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:40 smithi150 ceph-mon[65103]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:41.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:40 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:41.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:40 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:41.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:40 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T21:36:41.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:40 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:41.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:41 smithi130 ceph-mon[82829]: Reconfiguring osd.2 (monmap changed)... 2024-04-22T21:36:41.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:41 smithi130 ceph-mon[82829]: Reconfiguring daemon osd.2 on smithi130 2024-04-22T21:36:41.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:41.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:41.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T21:36:41.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:41.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:41 smithi130 ceph-mon[85205]: Reconfiguring osd.2 (monmap changed)... 2024-04-22T21:36:41.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:41 smithi130 ceph-mon[85205]: Reconfiguring daemon osd.2 on smithi130 2024-04-22T21:36:41.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:41.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:41.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T21:36:41.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:42.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:41 smithi150 ceph-mon[65103]: Reconfiguring osd.2 (monmap changed)... 2024-04-22T21:36:42.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:41 smithi150 ceph-mon[65103]: Reconfiguring daemon osd.2 on smithi130 2024-04-22T21:36:42.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:42.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:42.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T21:36:42.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:42.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:42 smithi130 ceph-mon[85205]: Reconfiguring osd.3 (monmap changed)... 2024-04-22T21:36:42.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:42 smithi130 ceph-mon[85205]: Reconfiguring daemon osd.3 on smithi130 2024-04-22T21:36:42.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:42 smithi130 ceph-mon[85205]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:42.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:42 smithi130 ceph-mon[82829]: Reconfiguring osd.3 (monmap changed)... 2024-04-22T21:36:42.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:42 smithi130 ceph-mon[82829]: Reconfiguring daemon osd.3 on smithi130 2024-04-22T21:36:42.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:42 smithi130 ceph-mon[82829]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:43.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:42 smithi150 ceph-mon[65103]: Reconfiguring osd.3 (monmap changed)... 2024-04-22T21:36:43.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:42 smithi150 ceph-mon[65103]: Reconfiguring daemon osd.3 on smithi130 2024-04-22T21:36:43.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:42 smithi150 ceph-mon[65103]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:44.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:44.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:44.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[82829]: Reconfiguring rgw.foo.smithi130.dfmola (monmap changed)... 2024-04-22T21:36:44.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi130.dfmola", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:36:44.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:44.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[82829]: Reconfiguring daemon rgw.foo.smithi130.dfmola on smithi130 2024-04-22T21:36:44.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[82829]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:44.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:44.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:44.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[85205]: Reconfiguring rgw.foo.smithi130.dfmola (monmap changed)... 2024-04-22T21:36:44.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi130.dfmola", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:36:44.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:44.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[85205]: Reconfiguring daemon rgw.foo.smithi130.dfmola on smithi130 2024-04-22T21:36:44.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:44 smithi130 ceph-mon[85205]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:44.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:44 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:44.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:44 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:44.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:44 smithi150 ceph-mon[65103]: Reconfiguring rgw.foo.smithi130.dfmola (monmap changed)... 2024-04-22T21:36:44.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:44 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi130.dfmola", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:36:44.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:44 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:44.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:44 smithi150 ceph-mon[65103]: Reconfiguring daemon rgw.foo.smithi130.dfmola on smithi130 2024-04-22T21:36:44.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:44 smithi150 ceph-mon[65103]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:46.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:46.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:46.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[82829]: Reconfiguring mon.b (monmap changed)... 2024-04-22T21:36:46.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:36:46.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:36:46.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:46.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[82829]: Reconfiguring daemon mon.b on smithi150 2024-04-22T21:36:46.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[82829]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:46.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:46.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:46.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[85205]: Reconfiguring mon.b (monmap changed)... 2024-04-22T21:36:46.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:36:46.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:36:46.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:46.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[85205]: Reconfiguring daemon mon.b on smithi150 2024-04-22T21:36:46.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:46 smithi130 ceph-mon[85205]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:46.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:46 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:46.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:46 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:46.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:46 smithi150 ceph-mon[65103]: Reconfiguring mon.b (monmap changed)... 2024-04-22T21:36:46.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:46 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T21:36:46.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:46 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T21:36:46.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:46 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:46.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:46 smithi150 ceph-mon[65103]: Reconfiguring daemon mon.b on smithi150 2024-04-22T21:36:46.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:46 smithi150 ceph-mon[65103]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:47.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:47.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:47.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:36:47.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:36:47.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:48.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:48.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:48.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:36:48.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:36:48.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T21:36:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T21:36:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:49.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:48 smithi150 ceph-mon[65103]: Reconfiguring mgr.x (monmap changed)... 2024-04-22T21:36:49.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:48 smithi150 ceph-mon[65103]: Reconfiguring daemon mgr.x on smithi150 2024-04-22T21:36:49.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:48 smithi150 ceph-mon[65103]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:49.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:48 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:49.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:48 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:49.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:48 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T21:36:49.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:48 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:49.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:36:48] "GET /metrics HTTP/1.1" 200 37953 "" "Prometheus/2.43.0" 2024-04-22T21:36:49.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[85205]: Reconfiguring mgr.x (monmap changed)... 2024-04-22T21:36:49.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[85205]: Reconfiguring daemon mgr.x on smithi150 2024-04-22T21:36:49.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[85205]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:49.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:49.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:49.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T21:36:49.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:49.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[82829]: Reconfiguring mgr.x (monmap changed)... 2024-04-22T21:36:49.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[82829]: Reconfiguring daemon mgr.x on smithi150 2024-04-22T21:36:49.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[82829]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:49.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:49.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:49.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T21:36:49.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:48 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:50.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:49 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:50.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:49 smithi130 ceph-mon[82829]: Reconfiguring osd.4 (monmap changed)... 2024-04-22T21:36:50.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:49 smithi130 ceph-mon[82829]: Reconfiguring daemon osd.4 on smithi150 2024-04-22T21:36:50.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:49 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:50.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:49 smithi130 ceph-mon[85205]: Reconfiguring osd.4 (monmap changed)... 2024-04-22T21:36:50.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:49 smithi130 ceph-mon[85205]: Reconfiguring daemon osd.4 on smithi150 2024-04-22T21:36:50.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:49 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:50.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:49 smithi150 ceph-mon[65103]: Reconfiguring osd.4 (monmap changed)... 2024-04-22T21:36:50.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:49 smithi150 ceph-mon[65103]: Reconfiguring daemon osd.4 on smithi150 2024-04-22T21:36:50.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:50 smithi150 ceph-mon[65103]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:50.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:50 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:50.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:50 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:50.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:50 smithi150 ceph-mon[65103]: Reconfiguring osd.5 (monmap changed)... 2024-04-22T21:36:50.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:50 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T21:36:50.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:50 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:50.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:50 smithi150 ceph-mon[65103]: Reconfiguring daemon osd.5 on smithi150 2024-04-22T21:36:51.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[82829]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:51.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:51.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:51.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[82829]: Reconfiguring osd.5 (monmap changed)... 2024-04-22T21:36:51.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T21:36:51.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:51.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[82829]: Reconfiguring daemon osd.5 on smithi150 2024-04-22T21:36:51.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[85205]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:51.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:51.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:51.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[85205]: Reconfiguring osd.5 (monmap changed)... 2024-04-22T21:36:51.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T21:36:51.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:51.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:50 smithi130 ceph-mon[85205]: Reconfiguring daemon osd.5 on smithi150 2024-04-22T21:36:52.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:36:52.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:51 smithi130 ceph-mon[85205]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:52.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:36:52.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:51 smithi130 ceph-mon[82829]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:52.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:36:52.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:51 smithi150 ceph-mon[65103]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:53.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:53.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:53.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:52 smithi130 ceph-mon[82829]: Reconfiguring osd.6 (monmap changed)... 2024-04-22T21:36:53.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T21:36:53.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:53.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:52 smithi130 ceph-mon[82829]: Reconfiguring daemon osd.6 on smithi150 2024-04-22T21:36:53.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:53.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:53.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:52 smithi130 ceph-mon[85205]: Reconfiguring osd.6 (monmap changed)... 2024-04-22T21:36:53.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T21:36:53.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:53.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:52 smithi130 ceph-mon[85205]: Reconfiguring daemon osd.6 on smithi150 2024-04-22T21:36:53.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:53.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:53.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:52 smithi150 ceph-mon[65103]: Reconfiguring osd.6 (monmap changed)... 2024-04-22T21:36:53.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T21:36:53.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:53.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:52 smithi150 ceph-mon[65103]: Reconfiguring daemon osd.6 on smithi150 2024-04-22T21:36:54.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:54.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:54.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[82829]: Reconfiguring osd.7 (monmap changed)... 2024-04-22T21:36:54.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T21:36:54.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:54.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[82829]: Reconfiguring daemon osd.7 on smithi150 2024-04-22T21:36:54.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[82829]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:54.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:54.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:54.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[85205]: Reconfiguring osd.7 (monmap changed)... 2024-04-22T21:36:54.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T21:36:54.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:54.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[85205]: Reconfiguring daemon osd.7 on smithi150 2024-04-22T21:36:54.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:54 smithi130 ceph-mon[85205]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:54.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:54.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:54.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:54 smithi150 ceph-mon[65103]: Reconfiguring osd.7 (monmap changed)... 2024-04-22T21:36:54.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T21:36:54.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:54.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:54 smithi150 ceph-mon[65103]: Reconfiguring daemon osd.7 on smithi150 2024-04-22T21:36:54.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:54 smithi150 ceph-mon[65103]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:55.776 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:36:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[85205]: Reconfiguring rgw.foo.smithi150.xqvixd (monmap changed)... 2024-04-22T21:36:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi150.xqvixd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:36:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[85205]: Reconfiguring daemon rgw.foo.smithi150.xqvixd on smithi150 2024-04-22T21:36:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[85205]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:56.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:56.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:56.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[82829]: Reconfiguring rgw.foo.smithi150.xqvixd (monmap changed)... 2024-04-22T21:36:56.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi150.xqvixd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:36:56.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:56.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[82829]: Reconfiguring daemon rgw.foo.smithi150.xqvixd on smithi150 2024-04-22T21:36:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:55 smithi130 ceph-mon[82829]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:55 smithi150 ceph-mon[65103]: Reconfiguring rgw.foo.smithi150.xqvixd (monmap changed)... 2024-04-22T21:36:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi150.xqvixd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:36:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:55 smithi150 ceph-mon[65103]: Reconfiguring daemon rgw.foo.smithi150.xqvixd on smithi150 2024-04-22T21:36:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:55 smithi150 ceph-mon[65103]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:36:56.427 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:36:56.427 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (3m) 30s ago 25m 28.6M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:36:56.427 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (2m) 45s ago 25m 83.7M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:36:56.427 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (2m) 30s ago 24m 52.5M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:36:56.427 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (4m) 45s ago 30m 448M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:36:56.427 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (14m) 30s ago 31m 492M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:36:56.427 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (59s) 30s ago 32m 41.6M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:36:56.427 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (108s) 45s ago 30m 38.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (35s) 30s ago 30m 19.9M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (4m) 30s ago 26m 16.0M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (4m) 45s ago 26m 15.8M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (30m) 30s ago 29m 55.4M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (29m) 30s ago 29m 57.6M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (29m) 30s ago 29m 50.6M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (28m) 30s ago 28m 53.5M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (28m) 45s ago 28m 55.5M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (28m) 45s ago 28m 52.8M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (27m) 45s ago 27m 54.5M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (27m) 45s ago 27m 53.8M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (3m) 45s ago 26m 69.5M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (24m) 30s ago 24m 79.2M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:36:56.428 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (24m) 45s ago 24m 76.2M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:36:56.862 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:36:56.862 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:36:56.862 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:36:56.862 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:36:56.862 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:36:56.862 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:36:56.863 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:36:56.863 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:36:56.863 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:36:56.863 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:36:56.863 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:36:56.863 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:36:56.863 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:36:56.863 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:36:56.863 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-22T21:36:56.863 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 5 2024-04-22T21:36:56.863 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:36:56.863 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:36:57.228 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:36:57.228 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": null, 2024-04-22T21:36:57.228 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": false, 2024-04-22T21:36:57.229 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "", 2024-04-22T21:36:57.229 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:36:57.229 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": null, 2024-04-22T21:36:57.229 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "", 2024-04-22T21:36:57.229 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:36:57.229 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:36:57.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='client.54168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:57.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='client.64108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:57.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='client.54180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:57.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:36:57.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mon 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all crash 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mds 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:36:57.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T21:36:57.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:36:57.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='client.54168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:57.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='client.64108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:57.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='client.54180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:57.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mon 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all crash 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mds 2024-04-22T21:36:57.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:36:57.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T21:36:57.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:36:57.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all nfs 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all node-exporter 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all prometheus 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all alertmanager 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all grafana 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all loki 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all promtail 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Finalizing container_image settings 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:36:57.553 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: Upgrade: Complete! 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:36:57.554 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/2662623878' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:57.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:36:57.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:57.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:36:57.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.555 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all nfs 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all node-exporter 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all prometheus 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all alertmanager 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all grafana 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all loki 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all promtail 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Finalizing container_image settings 2024-04-22T21:36:57.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:36:57.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:36:57.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:36:57.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:36:57.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:36:57.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:36:57.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:36:57.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:36:57.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:36:57.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:36:57.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:36:57.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: Upgrade: Complete! 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:36:57.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/2662623878' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:57.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:36:57.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:57.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:36:57.560 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='client.54168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:57.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='client.64108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:57.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='client.54180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:57.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:36:57.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mon 2024-04-22T21:36:57.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-22T21:36:57.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-22T21:36:57.850 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all crash 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mds 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.851 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:36:57.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all nfs 2024-04-22T21:36:57.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.852 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all node-exporter 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all prometheus 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all alertmanager 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.853 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all grafana 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all loki 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all promtail 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Finalizing container_image settings 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:36:57.854 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:36:57.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:36:57.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:36:57.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:36:57.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:36:57.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:36:57.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:36:57.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:36:57.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:36:57.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: Upgrade: Complete! 2024-04-22T21:36:57.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:36:57.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:36:57.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/2662623878' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:36:57.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:57.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:36:57.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:57.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:36:57.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:36:57.857 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:36:58.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:58 smithi130 ceph-mon[82829]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:58.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:58 smithi130 ceph-mon[82829]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:58.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:58 smithi130 ceph-mon[85205]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:58.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:58 smithi130 ceph-mon[85205]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:58.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:58 smithi150 ceph-mon[65103]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:36:58.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:58 smithi150 ceph-mon[65103]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:36:59.295 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:36:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:36:58] "GET /metrics HTTP/1.1" 200 37959 "" "Prometheus/2.43.0" 2024-04-22T21:36:59.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:36:59 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:59.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:36:59 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:36:59.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:36:59 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:00.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:00 smithi150 ceph-mon[65103]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:00.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:00 smithi130 ceph-mon[85205]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:00.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:00 smithi130 ceph-mon[82829]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:02.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:01 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:02.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:01 smithi130 ceph-mon[82829]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:02.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:01 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:02.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:01 smithi130 ceph-mon[85205]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:02.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:01 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:02.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:01 smithi150 ceph-mon[65103]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:04.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:03 smithi130 ceph-mon[82829]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:04.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:03 smithi130 ceph-mon[85205]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:04.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:03 smithi150 ceph-mon[65103]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:05 smithi130 ceph-mon[85205]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:05 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:37:06.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:05 smithi130 ceph-mon[82829]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:06.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:05 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:37:06.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:05 smithi150 ceph-mon[65103]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:06.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:05 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:37:08.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:07 smithi130 ceph-mon[82829]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:07 smithi130 ceph-mon[85205]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:08.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:07 smithi150 ceph-mon[65103]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:09.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:08 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:09.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:08 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:09.046 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:37:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:37:08] "GET /metrics HTTP/1.1" 200 37887 "" "Prometheus/2.43.0" 2024-04-22T21:37:09.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:08 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:10.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:09 smithi130 ceph-mon[82829]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:10.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:09 smithi130 ceph-mon[85205]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:09 smithi150 ceph-mon[65103]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:12.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:11 smithi130 ceph-mon[82829]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:12.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:11 smithi130 ceph-mon[85205]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:12.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:11 smithi150 ceph-mon[65103]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:14.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:13 smithi130 ceph-mon[82829]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:14.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:13 smithi130 ceph-mon[85205]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:14.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:13 smithi150 ceph-mon[65103]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:16.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:15 smithi130 ceph-mon[82829]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:16.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:15 smithi130 ceph-mon[85205]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:16.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:15 smithi150 ceph-mon[65103]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:18.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:17 smithi130 ceph-mon[85205]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:18.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:17 smithi130 ceph-mon[82829]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:18.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:17 smithi150 ceph-mon[65103]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:19.001 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:18 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:19.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:37:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:37:18] "GET /metrics HTTP/1.1" 200 37887 "" "Prometheus/2.43.0" 2024-04-22T21:37:19.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:18 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:19.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:18 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:20.016 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:19 smithi150 ceph-mon[65103]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:20.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:19 smithi130 ceph-mon[82829]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:20.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:19 smithi130 ceph-mon[85205]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:21.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:20 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:37:21.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:20 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:37:21.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:20 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:37:22.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:21 smithi130 ceph-mon[82829]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:22.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:21 smithi130 ceph-mon[85205]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:22.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:21 smithi150 ceph-mon[65103]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:24.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:23 smithi130 ceph-mon[82829]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:24.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:23 smithi130 ceph-mon[85205]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:24.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:23 smithi150 ceph-mon[65103]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:26.008 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:25 smithi130 ceph-mon[82829]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:26.008 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:25 smithi130 ceph-mon[85205]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:26.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:25 smithi150 ceph-mon[65103]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:27.922 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:27 smithi130 ceph-mon[82829]: from='client.64132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:27.922 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:27 smithi130 ceph-mon[82829]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:27.923 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:27 smithi130 ceph-mon[85205]: from='client.64132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:27.923 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:27 smithi130 ceph-mon[85205]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:28.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:27 smithi150 ceph-mon[65103]: from='client.64132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:28.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:27 smithi150 ceph-mon[65103]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:28.306 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps' 2024-04-22T21:37:28.898 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:28 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:28.899 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:28 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:29.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:28 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:29.280 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:37:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:37:28] "GET /metrics HTTP/1.1" 200 37954 "" "Prometheus/2.43.0" 2024-04-22T21:37:29.921 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:29 smithi130 ceph-mon[82829]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:29.922 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:29 smithi130 ceph-mon[85205]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:30.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:29 smithi150 ceph-mon[65103]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:30.352 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:37:30.352 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (3m) 63s ago 26m 28.6M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:37:30.352 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (3m) 79s ago 25m 83.7M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:37:30.352 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (3m) 63s ago 25m 52.5M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:37:30.352 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (5m) 79s ago 30m 448M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:37:30.352 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (14m) 63s ago 32m 492M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:37:30.352 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (93s) 63s ago 32m 41.6M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:37:30.352 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (2m) 79s ago 31m 38.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:37:30.352 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (69s) 63s ago 31m 19.9M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:37:30.352 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (4m) 63s ago 26m 16.0M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:37:30.352 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (4m) 79s ago 26m 15.8M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:37:30.353 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (30m) 63s ago 30m 55.4M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:37:30.353 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (30m) 63s ago 30m 57.6M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:37:30.353 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (29m) 63s ago 29m 50.6M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:37:30.353 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (29m) 63s ago 29m 53.5M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:37:30.353 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (28m) 79s ago 28m 55.5M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:37:30.353 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (28m) 79s ago 28m 52.8M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:37:30.353 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (28m) 79s ago 28m 54.5M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:37:30.353 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (27m) 79s ago 27m 53.8M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:37:30.353 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (4m) 79s ago 27m 69.5M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:37:30.353 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (25m) 63s ago 25m 79.2M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:37:30.353 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (25m) 79s ago 25m 76.2M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:37:31.005 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:30 smithi130 ceph-mon[85205]: from='client.54204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:31.005 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:30 smithi130 ceph-mon[82829]: from='client.54204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:31.029 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-04-22T21:37:31.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:30 smithi150 ceph-mon[65103]: from='client.54204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:31.946 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:31 smithi130 ceph-mon[82829]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:31.946 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:31 smithi130 ceph-mon[85205]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:32.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:31 smithi150 ceph-mon[65103]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:33.088 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:37:33.370 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:33 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/2708725852' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:33.370 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:33 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/2708725852' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:33.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:33 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/2708725852' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:33.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-04-22T21:37:34.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:34 smithi130 ceph-mon[82829]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:34.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:34 smithi130 ceph-mon[85205]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:34.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:34 smithi150 ceph-mon[65103]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)" 2024-04-22T21:37:35.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:35 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/2167559534' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:35.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:35 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/2167559534' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:36.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:35 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/2167559534' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:36.249 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-04-22T21:37:36.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:36 smithi130 ceph-mon[85205]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:36.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:37:36.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:36 smithi130 ceph-mon[82829]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:36.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:37:37.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:36 smithi150 ceph-mon[65103]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:37.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:37:37.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:37 smithi130 ceph-mon[85205]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:37.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:37 smithi130 ceph-mon[82829]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:38.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:37 smithi150 ceph-mon[65103]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:39.016 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:37:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:37:38] "GET /metrics HTTP/1.1" 200 37954 "" "Prometheus/2.43.0" 2024-04-22T21:37:39.016 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:38 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:39.016 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:38 smithi130 ceph-mon[82829]: from='client.54222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:39.016 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:38 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:39.017 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:38 smithi130 ceph-mon[85205]: from='client.54222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:39.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:38 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:39.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:38 smithi150 ceph-mon[65103]: from='client.54222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:39.787 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:39 smithi130 ceph-mon[82829]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:39.787 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:39 smithi130 ceph-mon[85205]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:39.870 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:37:40.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:39 smithi150 ceph-mon[65103]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:40.431 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade status' 2024-04-22T21:37:41.877 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:41 smithi130 ceph-mon[82829]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:41.878 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:41 smithi130 ceph-mon[85205]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:42.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:41 smithi150 ceph-mon[65103]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:42.328 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:37:42.329 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": null, 2024-04-22T21:37:42.329 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": false, 2024-04-22T21:37:42.329 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "", 2024-04-22T21:37:42.329 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:37:42.329 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": null, 2024-04-22T21:37:42.329 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "", 2024-04-22T21:37:42.329 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:37:42.329 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:37:42.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph health detail' 2024-04-22T21:37:43.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:42 smithi130 ceph-mon[85205]: from='client.64162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:43.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:42 smithi130 ceph-mon[82829]: from='client.64162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:43.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:42 smithi150 ceph-mon[65103]: from='client.64162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:43.922 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:43 smithi130 ceph-mon[85205]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:43.923 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:43 smithi130 ceph-mon[82829]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:44.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:43 smithi150 ceph-mon[65103]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:45.035 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-04-22T21:37:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:45 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/1161537530' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:37:45.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:45 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/1161537530' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:37:45.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:45 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/1161537530' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:37:45.579 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-04-22T21:37:46.199 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:46 smithi130 ceph-mon[85205]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:46.200 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:46 smithi130 ceph-mon[82829]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:46.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:46 smithi150 ceph-mon[65103]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:48.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:47 smithi130 ceph-mon[85205]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:48.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:47 smithi130 ceph-mon[85205]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:48.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:47 smithi130 ceph-mon[82829]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:48.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:47 smithi130 ceph-mon[82829]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:48.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:47 smithi150 ceph-mon[65103]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:48.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:47 smithi150 ceph-mon[65103]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:48.997 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:48 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:48.998 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:37:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:37:48] "GET /metrics HTTP/1.1" 200 37954 "" "Prometheus/2.43.0" 2024-04-22T21:37:48.998 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:48 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:49.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:48 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:49.040 INFO:teuthology.orchestra.run.smithi130.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:37:49.808 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T21:37:50.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:50 smithi130 ceph-mon[82829]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:37:50.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:50 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:50.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:50 smithi130 ceph-mon[82829]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:50.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:50 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:37:50.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:50 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:37:50.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:50 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:50.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:50 smithi130 ceph-mon[85205]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:37:50.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:50 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:50.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:50 smithi130 ceph-mon[85205]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:50.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:50 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:37:50.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:50 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:37:50.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:50 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:50.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:50 smithi150 ceph-mon[65103]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:37:50.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:50 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:50.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:50 smithi150 ceph-mon[65103]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:50.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:50 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:37:50.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:50 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:37:50.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:50 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:51.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:51 smithi130 ceph-mon[82829]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:37:51.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:37:51.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:51 smithi130 ceph-mon[85205]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:37:51.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:37:51.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:51 smithi150 ceph-mon[65103]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:37:51.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:37:52.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[85205]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:52.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:52.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:37:52.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:52.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:52.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:52.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:52.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:52.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:52.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T21:37:52.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[82829]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:52.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:52.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:37:52.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:52.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:52.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:52.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:52.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:52.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:52.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T21:37:52.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:52 smithi150 ceph-mon[65103]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:52.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:52.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:37:52.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:52.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:52.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:52.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:52.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:52.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:52.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T21:37:53.068 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:37:53.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[82829]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:37:53.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[82829]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:37:53.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mgr 2024-04-22T21:37:53.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mon 2024-04-22T21:37:53.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all crash 2024-04-22T21:37:53.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[82829]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T21:37:53.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[82829]: Upgrade: osd.0 is safe to restart 2024-04-22T21:37:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[85205]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:37:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[85205]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:37:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mgr 2024-04-22T21:37:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mon 2024-04-22T21:37:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all crash 2024-04-22T21:37:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[85205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T21:37:53.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:53 smithi130 ceph-mon[85205]: Upgrade: osd.0 is safe to restart 2024-04-22T21:37:53.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:53 smithi150 ceph-mon[65103]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:37:53.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:53 smithi150 ceph-mon[65103]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:37:53.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:53 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mgr 2024-04-22T21:37:53.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:53 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mon 2024-04-22T21:37:53.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:53 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all crash 2024-04-22T21:37:53.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:53 smithi150 ceph-mon[65103]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T21:37:53.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:53 smithi150 ceph-mon[65103]: Upgrade: osd.0 is safe to restart 2024-04-22T21:37:53.753 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:37:53.753 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (4m) 87s ago 26m 28.6M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:37:53.753 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (3m) 103s ago 26m 83.7M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:37:53.753 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (3m) 87s ago 25m 52.5M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:37:53.753 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (5m) 103s ago 31m 448M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:37:53.753 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (15m) 87s ago 32m 492M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:37:53.754 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (116s) 87s ago 32m 41.6M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:37:53.754 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (2m) 103s ago 31m 38.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:37:53.754 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (92s) 87s ago 31m 19.9M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:37:53.754 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (5m) 87s ago 27m 16.0M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:37:53.754 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (5m) 103s ago 27m 15.8M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:37:53.754 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (30m) 87s ago 30m 55.4M 2992M 17.2.0 e1d6a67b021e c92125c8f22d 2024-04-22T21:37:53.754 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (30m) 87s ago 30m 57.6M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:37:53.754 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (30m) 87s ago 30m 50.6M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:37:53.754 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (29m) 87s ago 29m 53.5M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:37:53.754 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (29m) 103s ago 29m 55.5M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:37:53.754 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (28m) 103s ago 28m 52.8M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:37:53.754 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (28m) 103s ago 28m 54.5M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:37:53.754 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (28m) 103s ago 28m 53.8M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:37:53.755 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (4m) 103s ago 27m 69.5M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:37:53.755 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (25m) 87s ago 25m 79.2M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:37:53.755 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (25m) 103s ago 25m 76.2M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:37:54.204 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:37:54.204 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:37:54.204 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 5 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:37:54.205 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:37:54.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:54 smithi130 ceph-mon[85205]: from='client.64180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:54.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:54 smithi130 ceph-mon[85205]: from='client.64186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:54.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:54 smithi130 ceph-mon[85205]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:54.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:54 smithi130 ceph-mon[82829]: from='client.64180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:54.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:54 smithi130 ceph-mon[82829]: from='client.64186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:54.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:54 smithi130 ceph-mon[82829]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:54.560 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:37:54.560 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:37:54.560 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:37:54.560 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-04-22T21:37:54.560 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:37:54.560 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "0/8 daemons upgraded", 2024-04-22T21:37:54.560 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T21:37:54.560 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:37:54.560 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:37:54.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:54 smithi150 ceph-mon[65103]: from='client.64180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:54.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:54 smithi150 ceph-mon[65103]: from='client.64186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:54.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:54 smithi150 ceph-mon[65103]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:55.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[82829]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:55.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[82829]: Upgrade: Updating osd.0 2024-04-22T21:37:55.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:55.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T21:37:55.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/3773182257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:55.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:37:55.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[82829]: Deploying daemon osd.0 on smithi130 2024-04-22T21:37:55.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[82829]: from='client.54270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:55.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[85205]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:55.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[85205]: Upgrade: Updating osd.0 2024-04-22T21:37:55.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:55.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T21:37:55.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/3773182257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:55.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:37:55.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[85205]: Deploying daemon osd.0 on smithi130 2024-04-22T21:37:55.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:55 smithi130 ceph-mon[85205]: from='client.54270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:55.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:55 smithi150 ceph-mon[65103]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:55.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:55 smithi150 ceph-mon[65103]: Upgrade: Updating osd.0 2024-04-22T21:37:55.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:37:55.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T21:37:55.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:55 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/3773182257' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:37:55.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:37:55.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:55 smithi150 ceph-mon[65103]: Deploying daemon osd.0 on smithi130 2024-04-22T21:37:55.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:55 smithi150 ceph-mon[65103]: from='client.54270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:37:56.374 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:56 smithi130 ceph-mon[85205]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:56.374 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:56 smithi130 ceph-mon[82829]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:56.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:56 smithi150 ceph-mon[65103]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:37:57.824 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:37:57 smithi130 systemd[1]: Stopping Ceph osd.0 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:37:58.152 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:37:57 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0[29273]: 2024-04-22T21:37:57.841+0000 7f2ed80c9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:37:58.152 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:37:57 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0[29273]: 2024-04-22T21:37:57.841+0000 7f2ed80c9700 -1 osd.0 113 *** Got signal Terminated *** 2024-04-22T21:37:58.152 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:37:57 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0[29273]: 2024-04-22T21:37:57.841+0000 7f2ed80c9700 -1 osd.0 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:37:58.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:58 smithi130 ceph-mon[85205]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:58.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:58 smithi130 ceph-mon[85205]: osd.0 marked itself down and dead 2024-04-22T21:37:58.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:58 smithi130 ceph-mon[82829]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:58.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:58 smithi130 ceph-mon[82829]: osd.0 marked itself down and dead 2024-04-22T21:37:58.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:58 smithi150 ceph-mon[65103]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:37:58.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:58 smithi150 ceph-mon[65103]: osd.0 marked itself down and dead 2024-04-22T21:37:58.967 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:37:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:37:58] "GET /metrics HTTP/1.1" 200 37955 "" "Prometheus/2.43.0" 2024-04-22T21:37:58.967 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:37:58 smithi130 podman[92528]: 2024-04-22 21:37:58.734452948 +0000 UTC m=+1.003450842 container died c92125c8f22dc5f81c0f9d0ae42c47bf3edf9ad66ea6ece863b057995b876be0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-22T21:37:59.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:59 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:59.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:59 smithi130 ceph-mon[85205]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:37:59.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:37:59 smithi130 ceph-mon[85205]: osdmap e114: 8 total, 7 up, 8 in 2024-04-22T21:37:59.296 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:37:59 smithi130 podman[92528]: 2024-04-22 21:37:59.076073791 +0000 UTC m=+1.345071673 container cleanup c92125c8f22dc5f81c0f9d0ae42c47bf3edf9ad66ea6ece863b057995b876be0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, release=754, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, ceph=True, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8) 2024-04-22T21:37:59.296 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:37:59 smithi130 bash[92528]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0 2024-04-22T21:37:59.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:59 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:59.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:59 smithi130 ceph-mon[82829]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:37:59.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:37:59 smithi130 ceph-mon[82829]: osdmap e114: 8 total, 7 up, 8 in 2024-04-22T21:37:59.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:59 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:37:59.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:59 smithi150 ceph-mon[65103]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:37:59.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:37:59 smithi150 ceph-mon[65103]: osdmap e114: 8 total, 7 up, 8 in 2024-04-22T21:37:59.722 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:37:59 smithi130 podman[92546]: 2024-04-22 21:37:59.319151795 +0000 UTC m=+0.584958207 container remove c92125c8f22dc5f81c0f9d0ae42c47bf3edf9ad66ea6ece863b057995b876be0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, RELEASE=HEAD, io.buildah.version=1.19.8, name=centos-stream, ceph=True, vcs-type=git, io.openshift.expose-services=, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , architecture=x86_64, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_BRANCH=HEAD) 2024-04-22T21:37:59.987 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:37:59 smithi130 podman[92637]: 2024-04-22 21:37:59.644549597 +0000 UTC m=+0.018630898 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:37:59.987 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:37:59 smithi130 podman[92637]: 2024-04-22 21:37:59.770366718 +0000 UTC m=+0.144448051 container create 3bc59fc3eb7aeaf36340aa9b40d9c9e41513f136a59358790f757c39a8b4d474 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-56e81df, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:38:00.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:00 smithi130 ceph-mon[85205]: osdmap e115: 8 total, 7 up, 8 in 2024-04-22T21:38:00.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:00 smithi130 ceph-mon[85205]: pgmap v62: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T21:38:00.287 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:00 smithi130 podman[92637]: 2024-04-22 21:38:00.005923151 +0000 UTC m=+0.380004442 container init 3bc59fc3eb7aeaf36340aa9b40d9c9e41513f136a59358790f757c39a8b4d474 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:38:00.287 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:00 smithi130 podman[92637]: 2024-04-22 21:38:00.010470922 +0000 UTC m=+0.384552217 container start 3bc59fc3eb7aeaf36340aa9b40d9c9e41513f136a59358790f757c39a8b4d474 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-22T21:38:00.287 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:00 smithi130 podman[92637]: 2024-04-22 21:38:00.097649155 +0000 UTC m=+0.471730449 container attach 3bc59fc3eb7aeaf36340aa9b40d9c9e41513f136a59358790f757c39a8b4d474 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416) 2024-04-22T21:38:00.287 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:00 smithi130 ceph-mon[82829]: osdmap e115: 8 total, 7 up, 8 in 2024-04-22T21:38:00.288 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:00 smithi130 ceph-mon[82829]: pgmap v62: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T21:38:00.545 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:00 smithi130 podman[92637]: 2024-04-22 21:38:00.305555498 +0000 UTC m=+0.679636790 container died 3bc59fc3eb7aeaf36340aa9b40d9c9e41513f136a59358790f757c39a8b4d474 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, RELEASE=main-56e81df, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T21:38:00.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:00 smithi150 ceph-mon[65103]: osdmap e115: 8 total, 7 up, 8 in 2024-04-22T21:38:00.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:00 smithi150 ceph-mon[65103]: pgmap v62: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T21:38:01.295 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:00 smithi130 podman[92663]: 2024-04-22 21:38:00.959866064 +0000 UTC m=+0.648768800 container remove 3bc59fc3eb7aeaf36340aa9b40d9c9e41513f136a59358790f757c39a8b4d474 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-22T21:38:01.295 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:00 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-22T21:38:01.295 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:00 smithi130 systemd[1]: Stopped Ceph osd.0 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:38:01.295 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:00 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.0.service: Consumed 14.493s CPU time. 2024-04-22T21:38:01.296 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:01 smithi130 systemd[1]: Starting Ceph osd.0 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:38:01.737 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:01 smithi130 podman[92767]: 2024-04-22 21:38:01.315637847 +0000 UTC m=+0.018153040 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:38:01.737 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:01 smithi130 podman[92767]: 2024-04-22 21:38:01.428033466 +0000 UTC m=+0.130548641 container create 468b95a3706d4b145bb8860c47405676e7f9f4f7d0b51d790fa970776edfd884 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True) 2024-04-22T21:38:01.737 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:01 smithi130 podman[92767]: 2024-04-22 21:38:01.65547728 +0000 UTC m=+0.357992456 container init 468b95a3706d4b145bb8860c47405676e7f9f4f7d0b51d790fa970776edfd884 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-56e81df, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416) 2024-04-22T21:38:01.737 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:01 smithi130 podman[92767]: 2024-04-22 21:38:01.6587917 +0000 UTC m=+0.361306875 container start 468b95a3706d4b145bb8860c47405676e7f9f4f7d0b51d790fa970776edfd884 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-56e81df, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux ) 2024-04-22T21:38:02.045 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:01 smithi130 podman[92767]: 2024-04-22 21:38:01.755570863 +0000 UTC m=+0.458086038 container attach 468b95a3706d4b145bb8860c47405676e7f9f4f7d0b51d790fa970776edfd884 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-22T21:38:02.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:01 smithi130 ceph-mon[82829]: pgmap v63: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 66/627 objects degraded (10.526%) 2024-04-22T21:38:02.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:01 smithi130 ceph-mon[82829]: Health check failed: Degraded data redundancy: 66/627 objects degraded (10.526%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T21:38:02.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:01 smithi130 ceph-mon[85205]: pgmap v63: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 66/627 objects degraded (10.526%) 2024-04-22T21:38:02.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:01 smithi130 ceph-mon[85205]: Health check failed: Degraded data redundancy: 66/627 objects degraded (10.526%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T21:38:02.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:01 smithi150 ceph-mon[65103]: pgmap v63: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 66/627 objects degraded (10.526%) 2024-04-22T21:38:02.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:01 smithi150 ceph-mon[65103]: Health check failed: Degraded data redundancy: 66/627 objects degraded (10.526%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T21:38:02.465 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate[92782]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:38:02.465 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate[92782]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:38:02.465 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 bash[92767]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:38:02.465 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 bash[92767]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 bash[92767]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 bash[92767]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 bash[92767]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 bash[92767]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 bash[92767]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 bash[92767]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 bash[92767]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate[92782]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate[92782]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate[92782]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate[92782]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate[92782]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate[92782]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T21:38:02.466 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate[92782]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-22T21:38:02.467 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 podman[92767]: 2024-04-22 21:38:02.16759283 +0000 UTC m=+0.870108007 container died 468b95a3706d4b145bb8860c47405676e7f9f4f7d0b51d790fa970776edfd884 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:38:03.045 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:02 smithi130 podman[92918]: 2024-04-22 21:38:02.788457809 +0000 UTC m=+0.613102527 container remove 468b95a3706d4b145bb8860c47405676e7f9f4f7d0b51d790fa970776edfd884 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-22T21:38:03.045 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:03 smithi130 podman[92983]: 2024-04-22 21:38:02.90467221 +0000 UTC m=+0.018055943 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:38:03.045 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:03 smithi130 podman[92983]: 2024-04-22 21:38:03.01019868 +0000 UTC m=+0.123582456 container create de61cc1c93b805af234561fc5a63ce81c03762495326c97e110601c6511e82a3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-56e81df) 2024-04-22T21:38:03.545 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:03 smithi130 podman[92983]: 2024-04-22 21:38:03.254059386 +0000 UTC m=+0.367443119 container init de61cc1c93b805af234561fc5a63ce81c03762495326c97e110601c6511e82a3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, GIT_CLEAN=True, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux ) 2024-04-22T21:38:03.545 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:03 smithi130 podman[92983]: 2024-04-22 21:38:03.257293433 +0000 UTC m=+0.370677175 container start de61cc1c93b805af234561fc5a63ce81c03762495326c97e110601c6511e82a3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T21:38:03.545 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:03 smithi130 ceph-osd[93002]: -- 172.21.15.130:0/664893525 <== mon.1 v2:172.21.15.130:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561df47c1680 con 0x561df478ac00 2024-04-22T21:38:03.546 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:03 smithi130 bash[92983]: de61cc1c93b805af234561fc5a63ce81c03762495326c97e110601c6511e82a3 2024-04-22T21:38:03.546 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:03 smithi130 systemd[1]: Started Ceph osd.0 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:38:04.045 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0[92998]: 2024-04-22T21:38:03.811+0000 7f678c1ae7c0 -1 Falling back to public interface 2024-04-22T21:38:04.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:04 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:04.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:04 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:04.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:04 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:04.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:04 smithi130 ceph-mon[82829]: pgmap v64: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 66/627 objects degraded (10.526%) 2024-04-22T21:38:04.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:04 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:04.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:04 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:04.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:04 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:04.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:04 smithi130 ceph-mon[85205]: pgmap v64: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 66/627 objects degraded (10.526%) 2024-04-22T21:38:04.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:04 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:04.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:04 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:04.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:04 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:04.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:04 smithi150 ceph-mon[65103]: pgmap v64: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 66/627 objects degraded (10.526%) 2024-04-22T21:38:06.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:05 smithi130 ceph-mon[82829]: pgmap v65: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 66/627 objects degraded (10.526%) 2024-04-22T21:38:06.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:05 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:06.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:05 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:38:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:05 smithi130 ceph-mon[85205]: pgmap v65: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 66/627 objects degraded (10.526%) 2024-04-22T21:38:06.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:05 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:06.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:05 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:38:06.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:05 smithi150 ceph-mon[65103]: pgmap v65: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 66/627 objects degraded (10.526%) 2024-04-22T21:38:06.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:05 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:06.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:05 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:38:06.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:06 smithi130 ceph-mon[85205]: from='osd.0 [v2:172.21.15.130:6802/1298243779,v1:172.21.15.130:6803/1298243779]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T21:38:06.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:06 smithi130 ceph-mon[85205]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T21:38:06.774 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0[92998]: 2024-04-22T21:38:06.523+0000 7f678c1ae7c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-04-22T21:38:06.774 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0[92998]: 2024-04-22T21:38:06.560+0000 7f678c1ae7c0 -1 osd.0 113 log_to_monitors true 2024-04-22T21:38:06.774 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:06 smithi130 ceph-mon[82829]: from='osd.0 [v2:172.21.15.130:6802/1298243779,v1:172.21.15.130:6803/1298243779]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T21:38:06.774 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:06 smithi130 ceph-mon[82829]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T21:38:07.045 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:38:06 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0[92998]: 2024-04-22T21:38:06.791+0000 7f677d606700 -1 osd.0 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:38:07.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:06 smithi150 ceph-mon[65103]: from='osd.0 [v2:172.21.15.130:6802/1298243779,v1:172.21.15.130:6803/1298243779]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T21:38:07.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:06 smithi150 ceph-mon[65103]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T21:38:08.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:07 smithi130 ceph-mon[82829]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T21:38:08.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:07 smithi130 ceph-mon[82829]: osdmap e116: 8 total, 7 up, 8 in 2024-04-22T21:38:08.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:07 smithi130 ceph-mon[82829]: from='osd.0 [v2:172.21.15.130:6802/1298243779,v1:172.21.15.130:6803/1298243779]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:38:08.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:07 smithi130 ceph-mon[82829]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:38:08.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:07 smithi130 ceph-mon[82829]: pgmap v67: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 484 B/s rd, 0 op/s; 66/627 objects degraded (10.526%) 2024-04-22T21:38:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:07 smithi130 ceph-mon[85205]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T21:38:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:07 smithi130 ceph-mon[85205]: osdmap e116: 8 total, 7 up, 8 in 2024-04-22T21:38:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:07 smithi130 ceph-mon[85205]: from='osd.0 [v2:172.21.15.130:6802/1298243779,v1:172.21.15.130:6803/1298243779]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:38:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:07 smithi130 ceph-mon[85205]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:38:08.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:07 smithi130 ceph-mon[85205]: pgmap v67: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 484 B/s rd, 0 op/s; 66/627 objects degraded (10.526%) 2024-04-22T21:38:08.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:07 smithi150 ceph-mon[65103]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T21:38:08.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:07 smithi150 ceph-mon[65103]: osdmap e116: 8 total, 7 up, 8 in 2024-04-22T21:38:08.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:07 smithi150 ceph-mon[65103]: from='osd.0 [v2:172.21.15.130:6802/1298243779,v1:172.21.15.130:6803/1298243779]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:38:08.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:07 smithi150 ceph-mon[65103]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:38:08.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:07 smithi150 ceph-mon[65103]: pgmap v67: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 484 B/s rd, 0 op/s; 66/627 objects degraded (10.526%) 2024-04-22T21:38:09.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:38:08] "GET /metrics HTTP/1.1" 200 38084 "" "Prometheus/2.43.0" 2024-04-22T21:38:09.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[82829]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:38:09.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[82829]: osd.0 [v2:172.21.15.130:6802/1298243779,v1:172.21.15.130:6803/1298243779] boot 2024-04-22T21:38:09.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[82829]: osdmap e117: 8 total, 8 up, 8 in 2024-04-22T21:38:09.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:38:09.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:09.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:09.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:09.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[85205]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:38:09.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[85205]: osd.0 [v2:172.21.15.130:6802/1298243779,v1:172.21.15.130:6803/1298243779] boot 2024-04-22T21:38:09.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[85205]: osdmap e117: 8 total, 8 up, 8 in 2024-04-22T21:38:09.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:38:09.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:09.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:09.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:08 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:09.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:08 smithi150 ceph-mon[65103]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:38:09.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:08 smithi150 ceph-mon[65103]: osd.0 [v2:172.21.15.130:6802/1298243779,v1:172.21.15.130:6803/1298243779] boot 2024-04-22T21:38:09.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:08 smithi150 ceph-mon[65103]: osdmap e117: 8 total, 8 up, 8 in 2024-04-22T21:38:09.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:08 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T21:38:09.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:08 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:09.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:08 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:09.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:08 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:09.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:09 smithi130 ceph-mon[85205]: osdmap e118: 8 total, 8 up, 8 in 2024-04-22T21:38:09.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:09.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:09.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:09 smithi130 ceph-mon[85205]: pgmap v70: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 66/627 objects degraded (10.526%) 2024-04-22T21:38:09.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:09 smithi130 ceph-mon[82829]: osdmap e118: 8 total, 8 up, 8 in 2024-04-22T21:38:09.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:09.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:09.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:09 smithi130 ceph-mon[82829]: pgmap v70: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 66/627 objects degraded (10.526%) 2024-04-22T21:38:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:09 smithi150 ceph-mon[65103]: osdmap e118: 8 total, 8 up, 8 in 2024-04-22T21:38:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:09 smithi150 ceph-mon[65103]: pgmap v70: 161 pgs: 19 active+undersized+degraded, 39 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 66/627 objects degraded (10.526%) 2024-04-22T21:38:12.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:11 smithi130 ceph-mon[82829]: pgmap v71: 161 pgs: 8 active+undersized+degraded, 13 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 29/627 objects degraded (4.625%) 2024-04-22T21:38:12.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:11 smithi130 ceph-mon[82829]: Health check update: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded (PG_DEGRADED) 2024-04-22T21:38:12.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:11 smithi130 ceph-mon[85205]: pgmap v71: 161 pgs: 8 active+undersized+degraded, 13 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 29/627 objects degraded (4.625%) 2024-04-22T21:38:12.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:11 smithi130 ceph-mon[85205]: Health check update: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded (PG_DEGRADED) 2024-04-22T21:38:12.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:11 smithi150 ceph-mon[65103]: pgmap v71: 161 pgs: 8 active+undersized+degraded, 13 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 29/627 objects degraded (4.625%) 2024-04-22T21:38:12.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:11 smithi150 ceph-mon[65103]: Health check update: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded (PG_DEGRADED) 2024-04-22T21:38:12.545 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:38:12 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:38:12.276+0000 7fd7212f0700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-22T21:38:13.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:13.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:13.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:13.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:38:13.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:13.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:38:13.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:13.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:13.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:13.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T21:38:13.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[82829]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T21:38:13.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[82829]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-22T21:38:13.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:13.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:13.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:13.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:38:13.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:13.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:38:13.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:13.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:13.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:13.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T21:38:13.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[85205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T21:38:13.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:13 smithi130 ceph-mon[85205]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-22T21:38:13.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:13 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:13.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:13 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:13.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:13 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:13.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:13 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:38:13.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:13 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:13.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:13 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:38:13.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:13 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:13.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:13 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:13.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:13 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:13.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:13 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T21:38:13.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:13 smithi150 ceph-mon[65103]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T21:38:13.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:13 smithi150 ceph-mon[65103]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-22T21:38:14.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:14 smithi130 ceph-mon[82829]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 740 B/s rd, 0 op/s 2024-04-22T21:38:14.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:14 smithi130 ceph-mon[85205]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 740 B/s rd, 0 op/s 2024-04-22T21:38:14.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:14 smithi150 ceph-mon[65103]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 740 B/s rd, 0 op/s 2024-04-22T21:38:15.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:15 smithi130 ceph-mon[82829]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded) 2024-04-22T21:38:15.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:15 smithi130 ceph-mon[82829]: Cluster is now healthy 2024-04-22T21:38:15.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:15 smithi130 ceph-mon[85205]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded) 2024-04-22T21:38:15.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:15 smithi130 ceph-mon[85205]: Cluster is now healthy 2024-04-22T21:38:15.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:15 smithi150 ceph-mon[65103]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/627 objects degraded (4.625%), 8 pgs degraded) 2024-04-22T21:38:15.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:15 smithi150 ceph-mon[65103]: Cluster is now healthy 2024-04-22T21:38:16.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:16 smithi130 ceph-mon[82829]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T21:38:16.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:16 smithi130 ceph-mon[85205]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T21:38:16.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:16 smithi150 ceph-mon[65103]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T21:38:18.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:17 smithi130 ceph-mon[85205]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T21:38:18.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:17 smithi130 ceph-mon[82829]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T21:38:18.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:17 smithi150 ceph-mon[65103]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T21:38:19.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:18 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:19.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:18 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:19.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:18 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:19.046 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:38:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:38:18] "GET /metrics HTTP/1.1" 200 38084 "" "Prometheus/2.43.0" 2024-04-22T21:38:20.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:19 smithi130 ceph-mon[85205]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 939 B/s rd, 0 op/s 2024-04-22T21:38:20.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:19 smithi130 ceph-mon[82829]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 939 B/s rd, 0 op/s 2024-04-22T21:38:20.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:19 smithi150 ceph-mon[65103]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 939 B/s rd, 0 op/s 2024-04-22T21:38:22.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:22.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:38:22.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:22.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:21 smithi130 ceph-mon[82829]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:22.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:22.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:38:22.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:22.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:21 smithi130 ceph-mon[85205]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:22.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:22.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:38:22.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:22.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:21 smithi150 ceph-mon[65103]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:24.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:23 smithi130 ceph-mon[82829]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:38:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:23 smithi130 ceph-mon[85205]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:38:24.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:23 smithi150 ceph-mon[65103]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:38:24.925 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:38:25.583 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:38:25.583 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (4m) 17s ago 27m 27.7M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:38:25.583 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (4m) 2m ago 26m 83.7M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:38:25.583 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (4m) 17s ago 26m 56.0M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:38:25.583 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (6m) 2m ago 31m 448M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:38:25.583 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (15m) 17s ago 33m 504M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:38:25.583 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (2m) 17s ago 33m 49.2M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:38:25.583 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (3m) 2m ago 32m 38.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:38:25.583 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (2m) 17s ago 32m 37.7M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:38:25.583 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (5m) 17s ago 27m 16.0M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:38:25.583 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (5m) 2m ago 27m 15.8M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:38:25.583 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (22s) 17s ago 31m 14.8M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:38:25.584 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (31m) 17s ago 31m 57.1M 2992M 17.2.0 e1d6a67b021e 8dc7c6aa2e3b 2024-04-22T21:38:25.584 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (30m) 17s ago 30m 51.1M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:38:25.584 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (30m) 17s ago 30m 53.9M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:38:25.584 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (29m) 2m ago 29m 55.5M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:38:25.584 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (29m) 2m ago 29m 52.8M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:38:25.584 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (29m) 2m ago 29m 54.5M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:38:25.584 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (28m) 2m ago 28m 53.8M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:38:25.584 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (5m) 2m ago 27m 69.5M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:38:25.584 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (26m) 17s ago 26m 79.6M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:38:25.584 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (26m) 2m ago 26m 76.2M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:38:25.980 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:38:25.980 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:38:25.980 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:38:25.980 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:38:25.980 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:38:25.980 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:38:25.980 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:38:25.980 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:38:25.980 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-22T21:38:25.980 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 1 2024-04-22T21:38:25.980 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:38:25.980 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:38:25.981 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:38:25.981 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:38:25.981 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:38:25.981 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-22T21:38:25.981 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 6 2024-04-22T21:38:25.981 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:38:25.981 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:38:25.981 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:25 smithi130 ceph-mon[82829]: from='client.64216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:25.981 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:25 smithi130 ceph-mon[82829]: from='client.54288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:25.981 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:25 smithi130 ceph-mon[82829]: from='client.54294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:25.982 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:25 smithi130 ceph-mon[82829]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:38:25.982 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:25 smithi130 ceph-mon[85205]: from='client.64216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:25.982 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:25 smithi130 ceph-mon[85205]: from='client.54288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:25.982 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:25 smithi130 ceph-mon[85205]: from='client.54294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:25.982 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:25 smithi130 ceph-mon[85205]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:38:26.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:25 smithi150 ceph-mon[65103]: from='client.64216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:26.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:25 smithi150 ceph-mon[65103]: from='client.54288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:26.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:25 smithi150 ceph-mon[65103]: from='client.54294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:26.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:25 smithi150 ceph-mon[65103]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:38:26.312 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:38:26.312 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:38:26.312 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:38:26.312 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-04-22T21:38:26.312 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:38:26.312 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "1/8 daemons upgraded", 2024-04-22T21:38:26.312 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T21:38:26.313 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:38:26.313 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:38:27.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:26 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/2940764093' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:27.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:26 smithi130 ceph-mon[82829]: from='client.54306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:27.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:26 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/2940764093' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:27.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:26 smithi130 ceph-mon[85205]: from='client.54306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:27.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:26 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/2940764093' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:27.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:26 smithi150 ceph-mon[65103]: from='client.54306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:28.001 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:27 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T21:38:28.001 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:27 smithi130 ceph-mon[82829]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T21:38:28.001 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:27 smithi130 ceph-mon[82829]: Upgrade: osd.1 is safe to restart 2024-04-22T21:38:28.001 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:27 smithi130 ceph-mon[82829]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:28.001 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:27 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T21:38:28.001 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:27 smithi130 ceph-mon[85205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T21:38:28.001 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:27 smithi130 ceph-mon[85205]: Upgrade: osd.1 is safe to restart 2024-04-22T21:38:28.001 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:27 smithi130 ceph-mon[85205]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:28.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:27 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T21:38:28.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:27 smithi150 ceph-mon[65103]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T21:38:28.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:27 smithi150 ceph-mon[65103]: Upgrade: osd.1 is safe to restart 2024-04-22T21:38:28.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:27 smithi150 ceph-mon[65103]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:29.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:28 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:29.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:38:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:38:28] "GET /metrics HTTP/1.1" 200 38102 "" "Prometheus/2.43.0" 2024-04-22T21:38:29.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:28 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:29.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:28 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:30.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:29 smithi130 ceph-mon[85205]: Upgrade: Updating osd.1 2024-04-22T21:38:30.174 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:29 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:30.175 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:29 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T21:38:30.175 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:29 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:30.175 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:29 smithi130 ceph-mon[85205]: Deploying daemon osd.1 on smithi130 2024-04-22T21:38:30.175 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:29 smithi130 ceph-mon[85205]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:38:30.175 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:29 smithi130 ceph-mon[82829]: Upgrade: Updating osd.1 2024-04-22T21:38:30.175 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:29 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:30.175 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:29 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T21:38:30.175 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:29 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:30.175 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:29 smithi130 ceph-mon[82829]: Deploying daemon osd.1 on smithi130 2024-04-22T21:38:30.175 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:29 smithi130 ceph-mon[82829]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:38:30.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:29 smithi150 ceph-mon[65103]: Upgrade: Updating osd.1 2024-04-22T21:38:30.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:29 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:30.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:29 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T21:38:30.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:29 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:30.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:29 smithi150 ceph-mon[65103]: Deploying daemon osd.1 on smithi130 2024-04-22T21:38:30.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:29 smithi150 ceph-mon[65103]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:38:31.866 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:31 smithi130 ceph-mon[82829]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:31.866 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:31 smithi130 ceph-mon[85205]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:32.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:31 smithi150 ceph-mon[65103]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:32.545 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:32 smithi130 systemd[1]: Stopping Ceph osd.1 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:38:32.545 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:32 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1[32250]: 2024-04-22T21:38:32.488+0000 7f073a855700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:38:32.546 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:32 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1[32250]: 2024-04-22T21:38:32.488+0000 7f073a855700 -1 osd.1 118 *** Got signal Terminated *** 2024-04-22T21:38:32.546 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:32 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1[32250]: 2024-04-22T21:38:32.488+0000 7f073a855700 -1 osd.1 118 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:38:33.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:32 smithi130 ceph-mon[85205]: osd.1 marked itself down and dead 2024-04-22T21:38:33.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:32 smithi130 ceph-mon[82829]: osd.1 marked itself down and dead 2024-04-22T21:38:33.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:32 smithi150 ceph-mon[65103]: osd.1 marked itself down and dead 2024-04-22T21:38:33.330 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:33 smithi130 podman[97464]: 2024-04-22 21:38:33.081267014 +0000 UTC m=+0.695403594 container died 8dc7c6aa2e3b98439e4d29d26d6eaad63d266890a37f413298288fcf122d9289 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1, io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.expose-services=, ceph=True, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-22T21:38:33.742 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:33 smithi130 podman[97464]: 2024-04-22 21:38:33.531304596 +0000 UTC m=+1.145441224 container cleanup 8dc7c6aa2e3b98439e4d29d26d6eaad63d266890a37f413298288fcf122d9289 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, version=8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, maintainer=Guillaume Abrioux , GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, com.redhat.component=centos-stream-container, architecture=x86_64, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8) 2024-04-22T21:38:33.742 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:33 smithi130 bash[97464]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1 2024-04-22T21:38:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:33 smithi130 ceph-mon[85205]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:38:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:33 smithi130 ceph-mon[85205]: osdmap e119: 8 total, 7 up, 8 in 2024-04-22T21:38:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:33 smithi130 ceph-mon[85205]: pgmap v83: 161 pgs: 19 stale+active+clean, 142 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:38:34.046 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:33 smithi130 podman[97481]: 2024-04-22 21:38:33.799303662 +0000 UTC m=+0.715625882 container remove 8dc7c6aa2e3b98439e4d29d26d6eaad63d266890a37f413298288fcf122d9289 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1, vcs-type=git, maintainer=Guillaume Abrioux , ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, architecture=x86_64, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T21:38:34.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:33 smithi130 ceph-mon[82829]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:38:34.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:33 smithi130 ceph-mon[82829]: osdmap e119: 8 total, 7 up, 8 in 2024-04-22T21:38:34.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:33 smithi130 ceph-mon[82829]: pgmap v83: 161 pgs: 19 stale+active+clean, 142 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:38:34.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:33 smithi150 ceph-mon[65103]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:38:34.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:33 smithi150 ceph-mon[65103]: osdmap e119: 8 total, 7 up, 8 in 2024-04-22T21:38:34.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:33 smithi150 ceph-mon[65103]: pgmap v83: 161 pgs: 19 stale+active+clean, 142 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:38:34.434 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:34 smithi130 podman[97577]: 2024-04-22 21:38:34.006941662 +0000 UTC m=+0.018207300 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:38:34.434 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:34 smithi130 podman[97577]: 2024-04-22 21:38:34.108714328 +0000 UTC m=+0.119979966 container create 647748152c4535c1c3b3c449ea08e05bd417ef7de64aaba7bf693413aa000c3a (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-22T21:38:34.434 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:34 smithi130 podman[97577]: 2024-04-22 21:38:34.327965781 +0000 UTC m=+0.339231420 container init 647748152c4535c1c3b3c449ea08e05bd417ef7de64aaba7bf693413aa000c3a (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:38:34.434 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:34 smithi130 podman[97577]: 2024-04-22 21:38:34.331254043 +0000 UTC m=+0.342519690 container start 647748152c4535c1c3b3c449ea08e05bd417ef7de64aaba7bf693413aa000c3a (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-deactivate, RELEASE=main-56e81df, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:38:34.434 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:34 smithi130 podman[97577]: 2024-04-22 21:38:34.452669917 +0000 UTC m=+0.463935570 container attach 647748152c4535c1c3b3c449ea08e05bd417ef7de64aaba7bf693413aa000c3a (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-22T21:38:34.755 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:34 smithi130 podman[97577]: 2024-04-22 21:38:34.63108837 +0000 UTC m=+0.642354000 container died 647748152c4535c1c3b3c449ea08e05bd417ef7de64aaba7bf693413aa000c3a (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-22T21:38:35.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:34 smithi130 ceph-mon[85205]: osdmap e120: 8 total, 7 up, 8 in 2024-04-22T21:38:35.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:34 smithi130 ceph-mon[82829]: osdmap e120: 8 total, 7 up, 8 in 2024-04-22T21:38:35.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:34 smithi150 ceph-mon[65103]: osdmap e120: 8 total, 7 up, 8 in 2024-04-22T21:38:35.545 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:35 smithi130 podman[97603]: 2024-04-22 21:38:35.181428673 +0000 UTC m=+0.545001662 container remove 647748152c4535c1c3b3c449ea08e05bd417ef7de64aaba7bf693413aa000c3a (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, ceph=True, GIT_CLEAN=True) 2024-04-22T21:38:35.545 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:35 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-22T21:38:35.545 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:35 smithi130 systemd[1]: Stopped Ceph osd.1 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:38:35.545 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:35 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.1.service: Consumed 14.491s CPU time. 2024-04-22T21:38:35.546 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:35 smithi130 systemd[1]: Starting Ceph osd.1 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:38:35.866 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:35 smithi130 ceph-mon[82829]: pgmap v85: 161 pgs: 19 stale+active+clean, 142 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T21:38:35.866 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:35.866 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:38:35.867 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:35 smithi130 ceph-mon[85205]: pgmap v85: 161 pgs: 19 stale+active+clean, 142 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T21:38:35.867 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:35.867 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:38:35.867 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:35 smithi130 podman[97703]: 2024-04-22 21:38:35.521446302 +0000 UTC m=+0.018769717 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:38:35.867 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:35 smithi130 podman[97703]: 2024-04-22 21:38:35.666190446 +0000 UTC m=+0.163513836 container create 4a35b07e1700faf22da56205a352feb49079e18bcad3bc7e81b4ff0729a9ef57 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=main-56e81df, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:38:36.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:35 smithi150 ceph-mon[65103]: pgmap v85: 161 pgs: 19 stale+active+clean, 142 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T21:38:36.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:36.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:38:36.295 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:35 smithi130 podman[97703]: 2024-04-22 21:38:35.885403889 +0000 UTC m=+0.382727284 container init 4a35b07e1700faf22da56205a352feb49079e18bcad3bc7e81b4ff0729a9ef57 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-22T21:38:36.295 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:35 smithi130 podman[97703]: 2024-04-22 21:38:35.888950826 +0000 UTC m=+0.386274221 container start 4a35b07e1700faf22da56205a352feb49079e18bcad3bc7e81b4ff0729a9ef57 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-22T21:38:36.295 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:35 smithi130 podman[97703]: 2024-04-22 21:38:35.968764186 +0000 UTC m=+0.466087585 container attach 4a35b07e1700faf22da56205a352feb49079e18bcad3bc7e81b4ff0729a9ef57 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:38:36.687 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate[97718]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:38:36.688 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate[97718]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:38:36.688 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 bash[97703]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:38:36.688 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 bash[97703]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:38:36.688 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 bash[97703]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T21:38:36.688 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 bash[97703]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T21:38:36.688 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 bash[97703]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T21:38:36.688 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 bash[97703]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T21:38:36.688 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 bash[97703]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-22T21:38:36.688 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 bash[97703]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T21:38:36.688 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 bash[97703]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-22T21:38:36.688 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate[97718]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T21:38:36.688 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate[97718]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T21:38:36.689 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate[97718]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T21:38:36.689 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate[97718]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T21:38:36.689 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate[97718]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-22T21:38:36.689 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate[97718]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T21:38:36.689 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate[97718]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-22T21:38:36.689 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 podman[97703]: 2024-04-22 21:38:36.393254538 +0000 UTC m=+0.890577930 container died 4a35b07e1700faf22da56205a352feb49079e18bcad3bc7e81b4ff0729a9ef57 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, RELEASE=main-56e81df, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:38:37.295 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:36 smithi130 podman[97860]: 2024-04-22 21:38:36.997861851 +0000 UTC m=+0.598869491 container remove 4a35b07e1700faf22da56205a352feb49079e18bcad3bc7e81b4ff0729a9ef57 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.build-date=20240416, RELEASE=main-56e81df, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-22T21:38:37.296 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:37 smithi130 podman[97928]: 2024-04-22 21:38:37.207144804 +0000 UTC m=+0.111313334 container create 3a370ce64feceff6250574d2622d7f19e0f73968ecf8e3415fd4854d58847147 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-22T21:38:37.296 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:37 smithi130 podman[97928]: 2024-04-22 21:38:37.114512235 +0000 UTC m=+0.018680729 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:38:37.795 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:37 smithi130 podman[97928]: 2024-04-22 21:38:37.450848031 +0000 UTC m=+0.355016515 container init 3a370ce64feceff6250574d2622d7f19e0f73968ecf8e3415fd4854d58847147 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-56e81df, ceph=True, org.label-schema.license=GPLv2) 2024-04-22T21:38:37.795 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:37 smithi130 podman[97928]: 2024-04-22 21:38:37.453877015 +0000 UTC m=+0.358045500 container start 3a370ce64feceff6250574d2622d7f19e0f73968ecf8e3415fd4854d58847147 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, ceph=True, org.label-schema.schema-version=1.0) 2024-04-22T21:38:37.795 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:37 smithi130 ceph-osd[97948]: -- 172.21.15.130:0/2297763803 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56228a3a7680 con 0x56228a370c00 2024-04-22T21:38:37.795 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:37 smithi130 bash[97928]: 3a370ce64feceff6250574d2622d7f19e0f73968ecf8e3415fd4854d58847147 2024-04-22T21:38:37.795 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:37 smithi130 systemd[1]: Started Ceph osd.1 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:38:38.295 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1[97944]: 2024-04-22T21:38:38.004+0000 7f35804a47c0 -1 Falling back to public interface 2024-04-22T21:38:38.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:38 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:38.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:38 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:38.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:38 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:38.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:38 smithi130 ceph-mon[82829]: pgmap v86: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-04-22T21:38:38.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:38 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:38.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:38 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:38.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:38 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:38.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:38 smithi130 ceph-mon[85205]: pgmap v86: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-04-22T21:38:38.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:38 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:38.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:38 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:38.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:38 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:38.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:38 smithi150 ceph-mon[65103]: pgmap v86: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-04-22T21:38:39.264 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:38:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:38:38] "GET /metrics HTTP/1.1" 200 38101 "" "Prometheus/2.43.0" 2024-04-22T21:38:39.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:39 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:39.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:39 smithi150 ceph-mon[65103]: Health check failed: Degraded data redundancy: 86/627 objects degraded (13.716%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T21:38:39.940 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:39 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:39.941 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:39 smithi130 ceph-mon[85205]: Health check failed: Degraded data redundancy: 86/627 objects degraded (13.716%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T21:38:39.941 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:39 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:39.941 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:39 smithi130 ceph-mon[82829]: Health check failed: Degraded data redundancy: 86/627 objects degraded (13.716%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T21:38:40.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:40 smithi150 ceph-mon[65103]: pgmap v87: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-04-22T21:38:41.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:40 smithi130 ceph-mon[82829]: pgmap v87: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-04-22T21:38:41.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:40 smithi130 ceph-mon[85205]: pgmap v87: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-04-22T21:38:41.047 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1[97944]: 2024-04-22T21:38:40.720+0000 7f35804a47c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-04-22T21:38:41.048 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:40 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1[97944]: 2024-04-22T21:38:40.748+0000 7f35804a47c0 -1 osd.1 118 log_to_monitors true 2024-04-22T21:38:41.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:41 smithi150 ceph-mon[65103]: from='osd.1 [v2:172.21.15.130:6810/3187843604,v1:172.21.15.130:6811/3187843604]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T21:38:41.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:41 smithi150 ceph-mon[65103]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T21:38:42.040 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:41 smithi130 ceph-mon[82829]: from='osd.1 [v2:172.21.15.130:6810/3187843604,v1:172.21.15.130:6811/3187843604]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T21:38:42.040 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:41 smithi130 ceph-mon[82829]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T21:38:42.041 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:41 smithi130 ceph-mon[85205]: from='osd.1 [v2:172.21.15.130:6810/3187843604,v1:172.21.15.130:6811/3187843604]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T21:38:42.041 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:41 smithi130 ceph-mon[85205]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T21:38:42.295 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1[97944]: 2024-04-22T21:38:42.058+0000 7f356d8f4700 -1 osd.1 118 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:38:43.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[82829]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T21:38:43.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[82829]: osdmap e121: 8 total, 7 up, 8 in 2024-04-22T21:38:43.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[82829]: from='osd.1 [v2:172.21.15.130:6810/3187843604,v1:172.21.15.130:6811/3187843604]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:38:43.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[82829]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:38:43.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[82829]: pgmap v89: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-04-22T21:38:43.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:43.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:43.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[82829]: from='osd.1 ' entity='osd.1' 2024-04-22T21:38:43.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[85205]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T21:38:43.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[85205]: osdmap e121: 8 total, 7 up, 8 in 2024-04-22T21:38:43.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[85205]: from='osd.1 [v2:172.21.15.130:6810/3187843604,v1:172.21.15.130:6811/3187843604]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:38:43.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[85205]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:38:43.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[85205]: pgmap v89: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-04-22T21:38:43.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:43.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:43.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:42 smithi130 ceph-mon[85205]: from='osd.1 ' entity='osd.1' 2024-04-22T21:38:43.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:42 smithi150 ceph-mon[65103]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T21:38:43.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:42 smithi150 ceph-mon[65103]: osdmap e121: 8 total, 7 up, 8 in 2024-04-22T21:38:43.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:42 smithi150 ceph-mon[65103]: from='osd.1 [v2:172.21.15.130:6810/3187843604,v1:172.21.15.130:6811/3187843604]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:38:43.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:42 smithi150 ceph-mon[65103]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:38:43.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:42 smithi150 ceph-mon[65103]: pgmap v89: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-04-22T21:38:43.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:42 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:43.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:42 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:43.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:42 smithi150 ceph-mon[65103]: from='osd.1 ' entity='osd.1' 2024-04-22T21:38:44.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:44.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:44.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[85205]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:38:44.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[85205]: osd.1 [v2:172.21.15.130:6810/3187843604,v1:172.21.15.130:6811/3187843604] boot 2024-04-22T21:38:44.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[85205]: osdmap e122: 8 total, 8 up, 8 in 2024-04-22T21:38:44.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:38:44.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[85205]: pgmap v91: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-04-22T21:38:44.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:44.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:44.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[82829]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:38:44.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[82829]: osd.1 [v2:172.21.15.130:6810/3187843604,v1:172.21.15.130:6811/3187843604] boot 2024-04-22T21:38:44.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[82829]: osdmap e122: 8 total, 8 up, 8 in 2024-04-22T21:38:44.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:38:44.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:44 smithi130 ceph-mon[82829]: pgmap v91: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-04-22T21:38:44.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:44 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:44.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:44 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:44.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:44 smithi150 ceph-mon[65103]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:38:44.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:44 smithi150 ceph-mon[65103]: osd.1 [v2:172.21.15.130:6810/3187843604,v1:172.21.15.130:6811/3187843604] boot 2024-04-22T21:38:44.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:44 smithi150 ceph-mon[65103]: osdmap e122: 8 total, 8 up, 8 in 2024-04-22T21:38:44.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:44 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T21:38:44.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:44 smithi150 ceph-mon[65103]: pgmap v91: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 86/627 objects degraded (13.716%) 2024-04-22T21:38:45.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:45 smithi130 ceph-mon[82829]: osdmap e123: 8 total, 8 up, 8 in 2024-04-22T21:38:45.244 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:45 smithi130 ceph-mon[85205]: osdmap e123: 8 total, 8 up, 8 in 2024-04-22T21:38:45.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:45 smithi150 ceph-mon[65103]: osdmap e123: 8 total, 8 up, 8 in 2024-04-22T21:38:46.199 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:46 smithi130 ceph-mon[82829]: pgmap v93: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 86/627 objects degraded (13.716%) 2024-04-22T21:38:46.199 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:46 smithi130 ceph-mon[85205]: pgmap v93: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 86/627 objects degraded (13.716%) 2024-04-22T21:38:46.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:46 smithi150 ceph-mon[65103]: pgmap v93: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 86/627 objects degraded (13.716%) 2024-04-22T21:38:48.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:38:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:38:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:38:48.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:38:48.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:38:48.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:38:48.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:38:48.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:38:48.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:38:48.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:38:48.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:38:48.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:38:48.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:38:48.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:38:48.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:38:48.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:38:48.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:38:48.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:38:48.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:38:48.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:38:48.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:38:48.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:38:48.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:38:48.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:38:48.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:38:48.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:38:48.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:38:48.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:48.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:38:48.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:48.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:38:48.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:38:48.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:38:48.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:48.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:38:48.053 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:47 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:48.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:38:48.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:38:48.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:38:48.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:38:48.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:38:48.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:38:48.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:47 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:48.999 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:48 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:38:49.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:48 smithi150 ceph-mon[65103]: Upgrade: Finalizing container_image settings 2024-04-22T21:38:49.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:48 smithi150 ceph-mon[65103]: Upgrade: Complete! 2024-04-22T21:38:49.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:48 smithi150 ceph-mon[65103]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T21:38:49.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:48 smithi150 ceph-mon[65103]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 86/627 objects degraded (13.716%), 23 pgs degraded) 2024-04-22T21:38:49.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:48 smithi150 ceph-mon[65103]: Cluster is now healthy 2024-04-22T21:38:49.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:38:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:38:48] "GET /metrics HTTP/1.1" 200 38101 "" "Prometheus/2.43.0" 2024-04-22T21:38:49.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:48 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:38:49.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:48 smithi130 ceph-mon[82829]: Upgrade: Finalizing container_image settings 2024-04-22T21:38:49.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:48 smithi130 ceph-mon[82829]: Upgrade: Complete! 2024-04-22T21:38:49.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:48 smithi130 ceph-mon[82829]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T21:38:49.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:48 smithi130 ceph-mon[82829]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 86/627 objects degraded (13.716%), 23 pgs degraded) 2024-04-22T21:38:49.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:48 smithi130 ceph-mon[82829]: Cluster is now healthy 2024-04-22T21:38:49.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:48 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:38:49.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:48 smithi130 ceph-mon[85205]: Upgrade: Finalizing container_image settings 2024-04-22T21:38:49.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:48 smithi130 ceph-mon[85205]: Upgrade: Complete! 2024-04-22T21:38:49.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:48 smithi130 ceph-mon[85205]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T21:38:49.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:48 smithi130 ceph-mon[85205]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 86/627 objects degraded (13.716%), 23 pgs degraded) 2024-04-22T21:38:49.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:48 smithi130 ceph-mon[85205]: Cluster is now healthy 2024-04-22T21:38:50.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:49 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:49 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:50.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:49 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:51.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:50 smithi130 ceph-mon[82829]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T21:38:51.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:50 smithi130 ceph-mon[85205]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T21:38:51.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:50 smithi150 ceph-mon[65103]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T21:38:52.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:52.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:38:52.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:52.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:51 smithi130 ceph-mon[85205]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:52.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:52.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:38:52.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:52.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:51 smithi130 ceph-mon[82829]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:52.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:52.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:38:52.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:52.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:51 smithi150 ceph-mon[65103]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:54.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:53 smithi130 ceph-mon[82829]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:38:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:53 smithi130 ceph-mon[85205]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:38:54.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:53 smithi150 ceph-mon[65103]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:38:56.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:55 smithi130 ceph-mon[82829]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 880 B/s rd, 0 op/s 2024-04-22T21:38:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:55 smithi130 ceph-mon[85205]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 880 B/s rd, 0 op/s 2024-04-22T21:38:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:55 smithi150 ceph-mon[65103]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 880 B/s rd, 0 op/s 2024-04-22T21:38:57.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:57 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:57.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:57 smithi130 ceph-mon[85205]: from='client.64252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:57.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:57 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:57.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:57 smithi130 ceph-mon[82829]: from='client.64252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:57.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:57 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:38:57.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:57 smithi150 ceph-mon[65103]: from='client.64252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:38:57.454 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps' 2024-04-22T21:38:58.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:58 smithi130 ceph-mon[82829]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:58.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:58 smithi130 ceph-mon[85205]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:58.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:58 smithi150 ceph-mon[65103]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:38:59.115 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:38:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:38:58] "GET /metrics HTTP/1.1" 200 38104 "" "Prometheus/2.43.0" 2024-04-22T21:38:59.115 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:38:59 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:59.116 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:38:59 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:59.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:38:59 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:38:59.561 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:38:59.561 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (5m) 17s ago 27m 27.7M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:38:59.561 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (4m) 2m ago 27m 83.7M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:38:59.561 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (4m) 17s ago 26m 58.1M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:38:59.561 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (6m) 2m ago 32m 448M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:38:59.561 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (16m) 17s ago 33m 506M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:38:59.561 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (3m) 17s ago 34m 50.4M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:38:59.561 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (3m) 2m ago 32m 38.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (2m) 17s ago 32m 38.4M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (6m) 17s ago 28m 16.1M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (6m) 2m ago 28m 15.8M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (56s) 17s ago 32m 45.2M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (22s) 17s ago 31m 12.7M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (31m) 17s ago 31m 51.7M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (30m) 17s ago 30m 54.5M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (30m) 2m ago 30m 55.5M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (30m) 2m ago 30m 52.8M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (29m) 2m ago 29m 54.5M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (29m) 2m ago 29m 53.8M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (5m) 2m ago 28m 69.5M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (26m) 17s ago 26m 79.7M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:38:59.562 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (26m) 2m ago 26m 76.2M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:39:00.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:00 smithi130 ceph-mon[85205]: from='client.64258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:00.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:00 smithi130 ceph-mon[85205]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:00.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:00 smithi130 ceph-mon[82829]: from='client.64258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:00.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:00 smithi130 ceph-mon[82829]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:00.326 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-04-22T21:39:00.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:00 smithi150 ceph-mon[65103]: from='client.64258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:00.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:00 smithi150 ceph-mon[65103]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:01.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:01 smithi130 ceph-mon[85205]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:01.849 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:01 smithi130 ceph-mon[82829]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:02.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:01 smithi150 ceph-mon[65103]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:02.370 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:39:03.016 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:02 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/2436701022' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:03.017 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:02 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/2436701022' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:03.041 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-04-22T21:39:03.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:02 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/2436701022' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:04.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:03 smithi130 ceph-mon[82829]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:03 smithi130 ceph-mon[85205]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:04.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:03 smithi150 ceph-mon[65103]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:05.932 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:05 smithi130 ceph-mon[85205]: from='client.54336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:05.932 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:05 smithi130 ceph-mon[85205]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:05.932 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:05 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:39:05.932 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:05 smithi130 ceph-mon[82829]: from='client.54336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:05.932 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:05 smithi130 ceph-mon[82829]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:05.933 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:05 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:39:06.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:05 smithi150 ceph-mon[65103]: from='client.54336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:06.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:05 smithi150 ceph-mon[65103]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:06.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:05 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:39:06.678 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:39:07.309 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade status' 2024-04-22T21:39:07.942 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:07 smithi130 ceph-mon[82829]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:07.942 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:07 smithi130 ceph-mon[85205]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:08.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:07 smithi150 ceph-mon[65103]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:09.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:08 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:09.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:08 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:09.046 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:39:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:39:08] "GET /metrics HTTP/1.1" 200 38107 "" "Prometheus/2.43.0" 2024-04-22T21:39:09.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:08 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:09.557 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:39:09.558 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": null, 2024-04-22T21:39:09.558 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": false, 2024-04-22T21:39:09.558 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "", 2024-04-22T21:39:09.558 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:39:09.558 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": null, 2024-04-22T21:39:09.558 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "", 2024-04-22T21:39:09.558 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:39:09.558 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:39:09.889 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:09 smithi130 ceph-mon[85205]: from='client.64276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:09.889 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:09 smithi130 ceph-mon[85205]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:09.889 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:09 smithi130 ceph-mon[82829]: from='client.64276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:09.889 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:09 smithi130 ceph-mon[82829]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:10.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:09 smithi150 ceph-mon[65103]: from='client.64276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:09 smithi150 ceph-mon[65103]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:10.177 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph health detail' 2024-04-22T21:39:12.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:11 smithi130 ceph-mon[82829]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:12.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:11 smithi130 ceph-mon[85205]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:12.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:11 smithi150 ceph-mon[65103]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:12.263 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-04-22T21:39:13.007 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:12 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/3059001200' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:39:13.008 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:12 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/3059001200' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:39:13.032 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-04-22T21:39:13.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:12 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/3059001200' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:39:14.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:13 smithi130 ceph-mon[82829]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:13 smithi130 ceph-mon[85205]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:14.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:13 smithi150 ceph-mon[65103]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:15.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:15 smithi130 ceph-mon[82829]: from='client.54354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:15.871 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:15 smithi130 ceph-mon[82829]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:15.871 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:15 smithi130 ceph-mon[85205]: from='client.54354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:15.871 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:15 smithi130 ceph-mon[85205]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:16.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:15 smithi150 ceph-mon[65103]: from='client.54354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:16.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:15 smithi150 ceph-mon[65103]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:16.607 INFO:teuthology.orchestra.run.smithi130.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:39:17.236 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T21:39:17.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:17 smithi130 ceph-mon[85205]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:39:17.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:17.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:17.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:39:17.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:17.874 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:17 smithi130 ceph-mon[82829]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:39:17.874 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:17.874 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:17.874 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:39:17.874 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:18.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:17 smithi150 ceph-mon[65103]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:39:18.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:18.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:18.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:39:18.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:18.897 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:18 smithi130 ceph-mon[85205]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:39:18.897 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:18 smithi130 ceph-mon[85205]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:18.897 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:18 smithi130 ceph-mon[82829]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:39:18.897 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:18 smithi130 ceph-mon[82829]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:19.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:18 smithi150 ceph-mon[65103]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:39:19.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:18 smithi150 ceph-mon[65103]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:19.295 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:39:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:39:18] "GET /metrics HTTP/1.1" 200 38107 "" "Prometheus/2.43.0" 2024-04-22T21:39:19.680 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:19 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:19.680 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:19 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:20.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:19 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:20.434 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:39:20.914 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[85205]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:20.914 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:20.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:39:20.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:20.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:20.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:20.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:20.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:20.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:20.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T21:39:20.915 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[82829]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:20.915 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:20.915 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:39:20.915 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:20 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T21:39:21.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:20 smithi150 ceph-mon[65103]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:21.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:20 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:21.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:20 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:39:21.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:20 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:21.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:20 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:21.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:20 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:21.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:20 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:21.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:20 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:21.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:20 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:21.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:20 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T21:39:21.121 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:39:21.121 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (5m) 39s ago 27m 27.7M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:39:21.121 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (5m) 3m ago 27m 83.7M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:39:21.121 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (5m) 39s ago 27m 58.1M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:39:21.121 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (7m) 3m ago 32m 448M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:39:21.121 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (16m) 39s ago 34m 506M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:39:21.121 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (3m) 39s ago 34m 50.4M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:39:21.121 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (4m) 3m ago 32m 38.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:39:21.121 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (3m) 39s ago 33m 38.4M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:39:21.122 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (6m) 39s ago 28m 16.1M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:39:21.122 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (6m) 3m ago 28m 15.8M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:39:21.122 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (78s) 39s ago 32m 45.2M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:39:21.122 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (44s) 39s ago 32m 12.7M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:39:21.122 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (31m) 39s ago 31m 51.7M 2992M 17.2.0 e1d6a67b021e 60205a8b4077 2024-04-22T21:39:21.122 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (31m) 39s ago 31m 54.5M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:39:21.122 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (30m) 3m ago 30m 55.5M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:39:21.122 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (30m) 3m ago 30m 52.8M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:39:21.122 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (30m) 3m ago 30m 54.5M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:39:21.122 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (29m) 3m ago 29m 53.8M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:39:21.122 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (6m) 3m ago 28m 69.5M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:39:21.122 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (27m) 39s ago 27m 79.7M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:39:21.122 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (27m) 3m ago 27m 76.2M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:39:21.523 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:39:21.523 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:39:21.523 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:39:21.523 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:39:21.523 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 7 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:39:21.524 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:39:21.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[85205]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:39:21.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[85205]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:39:21.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mgr 2024-04-22T21:39:21.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mon 2024-04-22T21:39:21.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all crash 2024-04-22T21:39:21.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[85205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T21:39:21.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[85205]: Upgrade: osd.2 is safe to restart 2024-04-22T21:39:21.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[85205]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:21.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:39:21.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/132812167' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:21.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[82829]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:39:21.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[82829]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:39:21.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mgr 2024-04-22T21:39:21.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mon 2024-04-22T21:39:21.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all crash 2024-04-22T21:39:21.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[82829]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T21:39:21.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[82829]: Upgrade: osd.2 is safe to restart 2024-04-22T21:39:21.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[82829]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:21.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:39:21.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:21 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/132812167' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:21.876 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:39:21.876 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:39:21.876 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:39:21.876 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-04-22T21:39:21.876 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:39:21.876 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "2/8 daemons upgraded", 2024-04-22T21:39:21.877 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T21:39:21.877 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:39:21.877 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:39:22.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:21 smithi150 ceph-mon[65103]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:39:22.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:21 smithi150 ceph-mon[65103]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:39:22.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:21 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mgr 2024-04-22T21:39:22.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:21 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mon 2024-04-22T21:39:22.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:21 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all crash 2024-04-22T21:39:22.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:21 smithi150 ceph-mon[65103]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T21:39:22.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:21 smithi150 ceph-mon[65103]: Upgrade: osd.2 is safe to restart 2024-04-22T21:39:22.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:21 smithi150 ceph-mon[65103]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:22.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:39:22.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:21 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/132812167' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:22.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[85205]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:22.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[85205]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:22.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[85205]: Upgrade: Updating osd.2 2024-04-22T21:39:22.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:22.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T21:39:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[85205]: Deploying daemon osd.2 on smithi130 2024-04-22T21:39:22.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[85205]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:22.892 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[82829]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:22.892 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[82829]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:22.892 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[82829]: Upgrade: Updating osd.2 2024-04-22T21:39:22.892 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:22.892 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T21:39:22.892 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:22.892 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[82829]: Deploying daemon osd.2 on smithi130 2024-04-22T21:39:22.892 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:22 smithi130 ceph-mon[82829]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:23.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:22 smithi150 ceph-mon[65103]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:23.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:22 smithi150 ceph-mon[65103]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:23.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:22 smithi150 ceph-mon[65103]: Upgrade: Updating osd.2 2024-04-22T21:39:23.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:22 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:23.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:22 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T21:39:23.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:22 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:23.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:22 smithi150 ceph-mon[65103]: Deploying daemon osd.2 on smithi130 2024-04-22T21:39:23.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:22 smithi150 ceph-mon[65103]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:23.753 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:23 smithi130 ceph-mon[82829]: from='client.64318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:23.753 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:23 smithi130 ceph-mon[85205]: from='client.64318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:24.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:23 smithi150 ceph-mon[65103]: from='client.64318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:24.986 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:24 smithi130 ceph-mon[85205]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:24.987 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:24 smithi130 ceph-mon[82829]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:25.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:24 smithi150 ceph-mon[65103]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:25.287 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:25 smithi130 systemd[1]: Stopping Ceph osd.2 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:39:25.545 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2[35178]: 2024-04-22T21:39:25.304+0000 7f21f9110700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:39:25.545 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2[35178]: 2024-04-22T21:39:25.304+0000 7f21f9110700 -1 osd.2 123 *** Got signal Terminated *** 2024-04-22T21:39:25.545 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2[35178]: 2024-04-22T21:39:25.304+0000 7f21f9110700 -1 osd.2 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:39:26.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:25 smithi130 ceph-mon[82829]: osd.2 marked itself down and dead 2024-04-22T21:39:26.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:25 smithi130 ceph-mon[85205]: osd.2 marked itself down and dead 2024-04-22T21:39:26.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:25 smithi150 ceph-mon[65103]: osd.2 marked itself down and dead 2024-04-22T21:39:26.387 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:26 smithi130 podman[103634]: 2024-04-22 21:39:26.067221436 +0000 UTC m=+0.903119607 container died 60205a8b40775879227abb581bb95898aa15f35e04c4ff41430fc338419f54a8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, distribution-scope=public, release=754, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, ceph=True, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, architecture=x86_64, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc.) 2024-04-22T21:39:26.681 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:26 smithi130 podman[103634]: 2024-04-22 21:39:26.504797511 +0000 UTC m=+1.340695683 container cleanup 60205a8b40775879227abb581bb95898aa15f35e04c4ff41430fc338419f54a8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2, name=centos-stream, RELEASE=HEAD, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, vcs-type=git, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, version=8, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, distribution-scope=public, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container) 2024-04-22T21:39:26.681 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:26 smithi130 bash[103634]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2 2024-04-22T21:39:27.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:26 smithi130 ceph-mon[82829]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:27.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:26 smithi130 ceph-mon[82829]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:39:27.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:26 smithi130 ceph-mon[82829]: osdmap e124: 8 total, 7 up, 8 in 2024-04-22T21:39:27.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:26 smithi130 ceph-mon[85205]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:27.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:26 smithi130 ceph-mon[85205]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:39:27.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:26 smithi130 ceph-mon[85205]: osdmap e124: 8 total, 7 up, 8 in 2024-04-22T21:39:27.045 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:26 smithi130 podman[103663]: 2024-04-22 21:39:26.785378177 +0000 UTC m=+0.720962989 container remove 60205a8b40775879227abb581bb95898aa15f35e04c4ff41430fc338419f54a8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2, name=centos-stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , release=754, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, version=8, ceph=True, architecture=x86_64, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-22T21:39:27.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:26 smithi150 ceph-mon[65103]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:27.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:26 smithi150 ceph-mon[65103]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:39:27.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:26 smithi150 ceph-mon[65103]: osdmap e124: 8 total, 7 up, 8 in 2024-04-22T21:39:27.525 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:27 smithi130 podman[103755]: 2024-04-22 21:39:27.1766835 +0000 UTC m=+0.018185218 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:39:27.525 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:27 smithi130 podman[103755]: 2024-04-22 21:39:27.299236298 +0000 UTC m=+0.140738003 container create c48e867aeeb0dff022eab811dafb12b177f2d41013d76949fc802403678bd3f0 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-56e81df, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-22T21:39:27.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:27 smithi130 ceph-mon[82829]: osdmap e125: 8 total, 7 up, 8 in 2024-04-22T21:39:27.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:27 smithi130 ceph-mon[85205]: osdmap e125: 8 total, 7 up, 8 in 2024-04-22T21:39:27.795 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:27 smithi130 podman[103755]: 2024-04-22 21:39:27.543806236 +0000 UTC m=+0.385307950 container init c48e867aeeb0dff022eab811dafb12b177f2d41013d76949fc802403678bd3f0 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-56e81df, io.buildah.version=1.33.5, ceph=True) 2024-04-22T21:39:27.795 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:27 smithi130 podman[103755]: 2024-04-22 21:39:27.547364251 +0000 UTC m=+0.388865957 container start c48e867aeeb0dff022eab811dafb12b177f2d41013d76949fc802403678bd3f0 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-deactivate, ceph=True, RELEASE=main-56e81df, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-22T21:39:27.795 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:27 smithi130 podman[103755]: 2024-04-22 21:39:27.618071808 +0000 UTC m=+0.459573529 container attach c48e867aeeb0dff022eab811dafb12b177f2d41013d76949fc802403678bd3f0 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-deactivate, RELEASE=main-56e81df, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-04-22T21:39:28.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:27 smithi150 ceph-mon[65103]: osdmap e125: 8 total, 7 up, 8 in 2024-04-22T21:39:28.118 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:27 smithi130 podman[103755]: 2024-04-22 21:39:27.843542244 +0000 UTC m=+0.685043958 container died c48e867aeeb0dff022eab811dafb12b177f2d41013d76949fc802403678bd3f0 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-22T21:39:28.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:28 smithi130 ceph-mon[82829]: pgmap v116: 161 pgs: 1 active+undersized+degraded, 39 peering, 6 active+undersized, 115 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 5/627 objects degraded (0.797%) 2024-04-22T21:39:28.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:28 smithi130 ceph-mon[82829]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-04-22T21:39:28.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:28 smithi130 ceph-mon[82829]: Health check failed: Degraded data redundancy: 5/627 objects degraded (0.797%), 1 pg degraded (PG_DEGRADED) 2024-04-22T21:39:28.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:28 smithi130 ceph-mon[85205]: pgmap v116: 161 pgs: 1 active+undersized+degraded, 39 peering, 6 active+undersized, 115 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 5/627 objects degraded (0.797%) 2024-04-22T21:39:28.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:28 smithi130 ceph-mon[85205]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-04-22T21:39:28.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:28 smithi130 ceph-mon[85205]: Health check failed: Degraded data redundancy: 5/627 objects degraded (0.797%), 1 pg degraded (PG_DEGRADED) 2024-04-22T21:39:28.796 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:28 smithi130 podman[103782]: 2024-04-22 21:39:28.480006225 +0000 UTC m=+0.631511346 container remove c48e867aeeb0dff022eab811dafb12b177f2d41013d76949fc802403678bd3f0 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-56e81df, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD) 2024-04-22T21:39:28.796 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:28 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-22T21:39:28.796 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:28 smithi130 systemd[1]: Stopped Ceph osd.2 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:39:28.796 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:28 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.2.service: Consumed 12.355s CPU time. 2024-04-22T21:39:28.796 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:28 smithi130 systemd[1]: Starting Ceph osd.2 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:39:29.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:28 smithi150 ceph-mon[65103]: pgmap v116: 161 pgs: 1 active+undersized+degraded, 39 peering, 6 active+undersized, 115 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 5/627 objects degraded (0.797%) 2024-04-22T21:39:29.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:28 smithi150 ceph-mon[65103]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-04-22T21:39:29.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:28 smithi150 ceph-mon[65103]: Health check failed: Degraded data redundancy: 5/627 objects degraded (0.797%), 1 pg degraded (PG_DEGRADED) 2024-04-22T21:39:29.172 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:39:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:39:28] "GET /metrics HTTP/1.1" 200 38106 "" "Prometheus/2.43.0" 2024-04-22T21:39:29.173 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:28 smithi130 podman[103883]: 2024-04-22 21:39:28.823513924 +0000 UTC m=+0.020025642 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:39:29.173 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:28 smithi130 podman[103883]: 2024-04-22 21:39:28.931469539 +0000 UTC m=+0.127981250 container create bb5f0dcae2b5d5a5a53698eceb72030a68d8e381620597714979eb0062400feb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, RELEASE=main-56e81df, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-22T21:39:29.545 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 podman[103883]: 2024-04-22 21:39:29.19202952 +0000 UTC m=+0.388541232 container init bb5f0dcae2b5d5a5a53698eceb72030a68d8e381620597714979eb0062400feb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-56e81df, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=) 2024-04-22T21:39:29.545 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 podman[103883]: 2024-04-22 21:39:29.19542511 +0000 UTC m=+0.391936823 container start bb5f0dcae2b5d5a5a53698eceb72030a68d8e381620597714979eb0062400feb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate, io.buildah.version=1.33.5, ceph=True, RELEASE=main-56e81df, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-22T21:39:29.545 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 podman[103883]: 2024-04-22 21:39:29.283740869 +0000 UTC m=+0.480252581 container attach bb5f0dcae2b5d5a5a53698eceb72030a68d8e381620597714979eb0062400feb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate, ceph=True, RELEASE=main-56e81df, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=) 2024-04-22T21:39:30.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:29 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:30.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:29 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:30.045 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate[103898]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:39:30.045 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate[103898]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 bash[103883]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 bash[103883]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 bash[103883]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 bash[103883]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 bash[103883]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 bash[103883]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 bash[103883]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 bash[103883]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 bash[103883]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate[103898]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate[103898]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate[103898]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate[103898]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T21:39:30.046 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate[103898]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-22T21:39:30.047 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate[103898]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T21:39:30.047 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate[103898]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-22T21:39:30.047 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:29 smithi130 podman[103883]: 2024-04-22 21:39:29.693898943 +0000 UTC m=+0.890410654 container died bb5f0dcae2b5d5a5a53698eceb72030a68d8e381620597714979eb0062400feb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-22T21:39:30.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:29 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:30.703 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:30 smithi130 podman[104040]: 2024-04-22 21:39:30.437374216 +0000 UTC m=+0.735412189 container remove bb5f0dcae2b5d5a5a53698eceb72030a68d8e381620597714979eb0062400feb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-activate, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=main-56e81df, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-22T21:39:30.703 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:30 smithi130 podman[104104]: 2024-04-22 21:39:30.556484931 +0000 UTC m=+0.018011457 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:39:30.703 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:30 smithi130 podman[104104]: 2024-04-22 21:39:30.663418936 +0000 UTC m=+0.124945465 container create 027e1ac8c1776484763e0d9835ceb8c47d3973c466603ec2edaefaa9c7cccabd (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, RELEASE=main-56e81df, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0) 2024-04-22T21:39:31.022 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:30 smithi130 ceph-mon[82829]: pgmap v117: 161 pgs: 1 active+undersized+degraded, 39 peering, 6 active+undersized, 115 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-04-22T21:39:31.022 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:30 smithi130 ceph-mon[85205]: pgmap v117: 161 pgs: 1 active+undersized+degraded, 39 peering, 6 active+undersized, 115 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-04-22T21:39:31.022 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:30 smithi130 podman[104104]: 2024-04-22 21:39:30.940700543 +0000 UTC m=+0.402227069 container init 027e1ac8c1776484763e0d9835ceb8c47d3973c466603ec2edaefaa9c7cccabd (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, RELEASE=main-56e81df, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-22T21:39:31.022 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:30 smithi130 podman[104104]: 2024-04-22 21:39:30.943668243 +0000 UTC m=+0.405194774 container start 027e1ac8c1776484763e0d9835ceb8c47d3973c466603ec2edaefaa9c7cccabd (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, RELEASE=main-56e81df, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5) 2024-04-22T21:39:31.023 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:30 smithi130 ceph-osd[104123]: -- 172.21.15.130:0/3992313611 <== mon.1 v2:172.21.15.130:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555b765a3680 con 0x555b7656d000 2024-04-22T21:39:31.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:30 smithi150 ceph-mon[65103]: pgmap v117: 161 pgs: 1 active+undersized+degraded, 39 peering, 6 active+undersized, 115 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 5/627 objects degraded (0.797%) 2024-04-22T21:39:31.295 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:31 smithi130 bash[104104]: 027e1ac8c1776484763e0d9835ceb8c47d3973c466603ec2edaefaa9c7cccabd 2024-04-22T21:39:31.295 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:31 smithi130 systemd[1]: Started Ceph osd.2 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:39:31.795 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:31 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2[104119]: 2024-04-22T21:39:31.485+0000 7f2c197737c0 -1 Falling back to public interface 2024-04-22T21:39:32.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:32.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:32.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:32.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:32 smithi130 ceph-mon[85205]: pgmap v118: 161 pgs: 11 active+undersized+degraded, 6 peering, 29 active+undersized, 115 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 36/627 objects degraded (5.742%) 2024-04-22T21:39:32.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:32.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:32.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:32.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:32 smithi130 ceph-mon[82829]: pgmap v118: 161 pgs: 11 active+undersized+degraded, 6 peering, 29 active+undersized, 115 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 36/627 objects degraded (5.742%) 2024-04-22T21:39:32.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:32.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:32.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:32.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:32 smithi150 ceph-mon[65103]: pgmap v118: 161 pgs: 11 active+undersized+degraded, 6 peering, 29 active+undersized, 115 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 36/627 objects degraded (5.742%) 2024-04-22T21:39:33.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:33 smithi130 ceph-mon[82829]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2024-04-22T21:39:33.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:33 smithi130 ceph-mon[85205]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2024-04-22T21:39:33.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:33 smithi150 ceph-mon[65103]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2024-04-22T21:39:34.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:34 smithi150 ceph-mon[65103]: pgmap v119: 161 pgs: 13 active+undersized+degraded, 33 active+undersized, 115 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 55/627 objects degraded (8.772%) 2024-04-22T21:39:34.392 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:34 smithi130 ceph-mon[82829]: pgmap v119: 161 pgs: 13 active+undersized+degraded, 33 active+undersized, 115 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 55/627 objects degraded (8.772%) 2024-04-22T21:39:34.393 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2[104119]: 2024-04-22T21:39:34.180+0000 7f2c197737c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-04-22T21:39:34.393 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2[104119]: 2024-04-22T21:39:34.209+0000 7f2c197737c0 -1 osd.2 123 log_to_monitors true 2024-04-22T21:39:34.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:34 smithi130 ceph-mon[85205]: pgmap v119: 161 pgs: 13 active+undersized+degraded, 33 active+undersized, 115 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 55/627 objects degraded (8.772%) 2024-04-22T21:39:35.295 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:39:35 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2[104119]: 2024-04-22T21:39:35.128+0000 7f2c0abcb700 -1 osd.2 123 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:39:35.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:35 smithi130 ceph-mon[82829]: Health check update: Degraded data redundancy: 55/627 objects degraded (8.772%), 13 pgs degraded (PG_DEGRADED) 2024-04-22T21:39:35.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:35 smithi130 ceph-mon[82829]: from='osd.2 [v2:172.21.15.130:6818/2692056286,v1:172.21.15.130:6819/2692056286]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T21:39:35.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:35 smithi130 ceph-mon[85205]: Health check update: Degraded data redundancy: 55/627 objects degraded (8.772%), 13 pgs degraded (PG_DEGRADED) 2024-04-22T21:39:35.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:35 smithi130 ceph-mon[85205]: from='osd.2 [v2:172.21.15.130:6818/2692056286,v1:172.21.15.130:6819/2692056286]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T21:39:35.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:35 smithi150 ceph-mon[65103]: Health check update: Degraded data redundancy: 55/627 objects degraded (8.772%), 13 pgs degraded (PG_DEGRADED) 2024-04-22T21:39:35.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:35 smithi150 ceph-mon[65103]: from='osd.2 [v2:172.21.15.130:6818/2692056286,v1:172.21.15.130:6819/2692056286]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T21:39:36.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[85205]: from='osd.2 [v2:172.21.15.130:6818/2692056286,v1:172.21.15.130:6819/2692056286]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T21:39:36.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[85205]: osdmap e126: 8 total, 7 up, 8 in 2024-04-22T21:39:36.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[85205]: from='osd.2 [v2:172.21.15.130:6818/2692056286,v1:172.21.15.130:6819/2692056286]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:39:36.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:36.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:36.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[85205]: pgmap v121: 161 pgs: 13 active+undersized+degraded, 33 active+undersized, 115 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 569 B/s rd, 0 op/s; 55/627 objects degraded (8.772%) 2024-04-22T21:39:36.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:36.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:39:36.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[82829]: from='osd.2 [v2:172.21.15.130:6818/2692056286,v1:172.21.15.130:6819/2692056286]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T21:39:36.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[82829]: osdmap e126: 8 total, 7 up, 8 in 2024-04-22T21:39:36.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[82829]: from='osd.2 [v2:172.21.15.130:6818/2692056286,v1:172.21.15.130:6819/2692056286]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:39:36.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:36.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:36.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[82829]: pgmap v121: 161 pgs: 13 active+undersized+degraded, 33 active+undersized, 115 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 569 B/s rd, 0 op/s; 55/627 objects degraded (8.772%) 2024-04-22T21:39:36.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:36.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:39:36.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:36 smithi150 ceph-mon[65103]: from='osd.2 [v2:172.21.15.130:6818/2692056286,v1:172.21.15.130:6819/2692056286]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T21:39:36.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:36 smithi150 ceph-mon[65103]: osdmap e126: 8 total, 7 up, 8 in 2024-04-22T21:39:36.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:36 smithi150 ceph-mon[65103]: from='osd.2 [v2:172.21.15.130:6818/2692056286,v1:172.21.15.130:6819/2692056286]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:39:36.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:36.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:36.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:36 smithi150 ceph-mon[65103]: pgmap v121: 161 pgs: 13 active+undersized+degraded, 33 active+undersized, 115 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 569 B/s rd, 0 op/s; 55/627 objects degraded (8.772%) 2024-04-22T21:39:36.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:36.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:39:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[85205]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:39:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[85205]: osd.2 [v2:172.21.15.130:6818/2692056286,v1:172.21.15.130:6819/2692056286] boot 2024-04-22T21:39:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[85205]: osdmap e127: 8 total, 8 up, 8 in 2024-04-22T21:39:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:39:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[85205]: osdmap e128: 8 total, 8 up, 8 in 2024-04-22T21:39:37.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[82829]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:39:37.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[82829]: osd.2 [v2:172.21.15.130:6818/2692056286,v1:172.21.15.130:6819/2692056286] boot 2024-04-22T21:39:37.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[82829]: osdmap e127: 8 total, 8 up, 8 in 2024-04-22T21:39:37.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:39:37.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:37.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:37.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:37 smithi130 ceph-mon[82829]: osdmap e128: 8 total, 8 up, 8 in 2024-04-22T21:39:37.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:37 smithi150 ceph-mon[65103]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:39:37.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:37 smithi150 ceph-mon[65103]: osd.2 [v2:172.21.15.130:6818/2692056286,v1:172.21.15.130:6819/2692056286] boot 2024-04-22T21:39:37.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:37 smithi150 ceph-mon[65103]: osdmap e127: 8 total, 8 up, 8 in 2024-04-22T21:39:37.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T21:39:37.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:37.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:37.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:37 smithi150 ceph-mon[65103]: osdmap e128: 8 total, 8 up, 8 in 2024-04-22T21:39:38.445 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:38 smithi130 ceph-mon[82829]: pgmap v124: 161 pgs: 33 peering, 13 active+undersized, 115 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:38.445 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:38 smithi130 ceph-mon[85205]: pgmap v124: 161 pgs: 33 peering, 13 active+undersized, 115 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:38.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:38 smithi150 ceph-mon[65103]: pgmap v124: 161 pgs: 33 peering, 13 active+undersized, 115 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:39.115 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:39:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:39:38] "GET /metrics HTTP/1.1" 200 37961 "" "Prometheus/2.43.0" 2024-04-22T21:39:39.512 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:39 smithi130 ceph-mon[85205]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 55/627 objects degraded (8.772%), 13 pgs degraded) 2024-04-22T21:39:39.512 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:39 smithi130 ceph-mon[85205]: Cluster is now healthy 2024-04-22T21:39:39.512 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:39 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:39.513 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:39 smithi130 ceph-mon[82829]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 55/627 objects degraded (8.772%), 13 pgs degraded) 2024-04-22T21:39:39.513 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:39 smithi130 ceph-mon[82829]: Cluster is now healthy 2024-04-22T21:39:39.513 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:39 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:39.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:39 smithi150 ceph-mon[65103]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 55/627 objects degraded (8.772%), 13 pgs degraded) 2024-04-22T21:39:39.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:39 smithi150 ceph-mon[65103]: Cluster is now healthy 2024-04-22T21:39:39.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:39 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:40.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:40 smithi130 ceph-mon[85205]: pgmap v125: 161 pgs: 33 peering, 13 active+undersized, 115 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:40.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:40 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:40.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:40 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:40.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:40 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:40.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:40 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:39:40.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:40 smithi130 ceph-mon[82829]: pgmap v125: 161 pgs: 33 peering, 13 active+undersized, 115 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:40.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:40 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:40.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:40 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:40.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:40 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:40.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:40 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:39:40.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:40 smithi150 ceph-mon[65103]: pgmap v125: 161 pgs: 33 peering, 13 active+undersized, 115 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:40.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:40 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:40.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:40 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:40.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:40 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:40.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:40 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:39:41.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:39:41.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: Upgrade: Finalizing container_image settings 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:39:41.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:39:41.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:39:41.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:39:41.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:39:41.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:39:41.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:39:41.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:39:41.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:39:41.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: Upgrade: Finalizing container_image settings 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:39:41.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:39:41.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:39:41.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:39:41.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:39:41.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:39:41.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:39:41.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.549 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:39:41.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:39:41.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:39:41.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:39:41.549 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: Upgrade: Complete! 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:41.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:39:41.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:39:41.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:39:41.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:39:41.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:39:41.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: Upgrade: Complete! 2024-04-22T21:39:41.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:39:41.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:39:41.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:41.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:39:41.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:41.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:39:41.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:39:41.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:39:41.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: Upgrade: Finalizing container_image settings 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:39:41.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:39:41.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:39:41.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:39:41.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:39:41.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:39:41.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.601 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:39:41.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: Upgrade: Complete! 2024-04-22T21:39:41.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:39:41.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:39:41.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:41.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:39:41.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:39:41.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:39:41.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:41.602 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:42.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:42 smithi130 ceph-mon[85205]: pgmap v126: 161 pgs: 9 peering, 152 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-22T21:39:42.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:42 smithi130 ceph-mon[82829]: pgmap v126: 161 pgs: 9 peering, 152 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-22T21:39:42.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:42 smithi150 ceph-mon[65103]: pgmap v126: 161 pgs: 9 peering, 152 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-22T21:39:44.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:43 smithi130 ceph-mon[82829]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:43 smithi130 ceph-mon[85205]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:44.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:43 smithi150 ceph-mon[65103]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:46.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:45 smithi130 ceph-mon[82829]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T21:39:46.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:45 smithi130 ceph-mon[85205]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T21:39:46.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:45 smithi150 ceph-mon[65103]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T21:39:48.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:47 smithi130 ceph-mon[82829]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 966 B/s rd, 0 op/s 2024-04-22T21:39:48.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:47 smithi130 ceph-mon[85205]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 966 B/s rd, 0 op/s 2024-04-22T21:39:48.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:47 smithi150 ceph-mon[65103]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 966 B/s rd, 0 op/s 2024-04-22T21:39:49.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:39:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:39:48] "GET /metrics HTTP/1.1" 200 37961 "" "Prometheus/2.43.0" 2024-04-22T21:39:49.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:48 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:49.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:48 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:49.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:48 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:49 smithi130 ceph-mon[85205]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:50.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:49 smithi130 ceph-mon[82829]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:50.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:49 smithi150 ceph-mon[65103]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:52.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:52.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:39:52.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:52.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:52.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:39:52.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:52.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:52.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:39:52.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:39:52.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:52 smithi130 ceph-mon[82829]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:52.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:52 smithi130 ceph-mon[85205]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:52.923 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps' 2024-04-22T21:39:53.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:52 smithi150 ceph-mon[65103]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:53.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:53 smithi130 ceph-mon[82829]: from='client.54396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:53.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:53 smithi130 ceph-mon[85205]: from='client.54396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:54.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:53 smithi150 ceph-mon[65103]: from='client.54396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:54.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:54 smithi130 ceph-mon[82829]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:54.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:54 smithi130 ceph-mon[85205]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:54.966 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:39:54.966 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (6m) 19s ago 28m 27.7M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:39:54.966 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (5m) 3m ago 28m 83.7M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (5m) 19s ago 27m 60.5M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (7m) 3m ago 33m 448M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (17m) 19s ago 34m 508M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (3m) 19s ago 35m 55.8M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (4m) 3m ago 33m 38.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (3m) 19s ago 33m 39.5M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (7m) 19s ago 29m 15.6M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (7m) 3m ago 29m 15.8M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (112s) 19s ago 32m 46.2M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (77s) 19s ago 32m 52.3M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (24s) 19s ago 32m 12.7M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 027e1ac8c177 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (31m) 19s ago 31m 56.1M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (31m) 3m ago 31m 55.5M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:39:54.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (30m) 3m ago 30m 52.8M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:39:54.968 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (30m) 3m ago 30m 54.5M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:39:54.968 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (30m) 3m ago 30m 53.8M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:39:54.968 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (6m) 3m ago 29m 69.5M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:39:54.968 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (27m) 19s ago 27m 79.9M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:39:54.968 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (27m) 3m ago 27m 76.2M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:39:55.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:54 smithi150 ceph-mon[65103]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:55.761 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-04-22T21:39:55.783 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:55 smithi130 ceph-mon[82829]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:55.783 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:55 smithi130 ceph-mon[82829]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:55.784 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:55 smithi130 ceph-mon[85205]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:55.784 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:55 smithi130 ceph-mon[85205]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:56.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:55 smithi150 ceph-mon[65103]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:39:56.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:55 smithi150 ceph-mon[65103]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:39:57.926 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:39:58.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:57 smithi130 ceph-mon[82829]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:58.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:57 smithi130 ceph-mon[85205]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:58.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:57 smithi150 ceph-mon[65103]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:39:58.783 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-04-22T21:39:59.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:39:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:39:58] "GET /metrics HTTP/1.1" 200 38119 "" "Prometheus/2.43.0" 2024-04-22T21:39:59.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:58 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/2707543184' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:59.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:58 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:58 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/2707543184' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:59.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:58 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:59.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:58 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/2707543184' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:39:59.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:58 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:39:59.761 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:39:59 smithi130 ceph-mon[82829]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:39:59 smithi130 ceph-mon[85205]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:00.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:39:59 smithi150 ceph-mon[65103]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:01.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:00 smithi130 ceph-mon[82829]: overall HEALTH_OK 2024-04-22T21:40:01.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:00 smithi130 ceph-mon[85205]: overall HEALTH_OK 2024-04-22T21:40:01.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:00 smithi150 ceph-mon[65103]: overall HEALTH_OK 2024-04-22T21:40:02.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:01 smithi130 ceph-mon[82829]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:02.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:01 smithi130 ceph-mon[82829]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:02.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:01 smithi130 ceph-mon[85205]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:02.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:01 smithi130 ceph-mon[85205]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:02.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:01 smithi150 ceph-mon[65103]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:02.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:01 smithi150 ceph-mon[65103]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:02.502 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:40:03.231 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade status' 2024-04-22T21:40:03.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:03 smithi130 ceph-mon[82829]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:03.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:03 smithi130 ceph-mon[85205]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:04.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:03 smithi150 ceph-mon[65103]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:05.236 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:40:05.236 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": null, 2024-04-22T21:40:05.236 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": false, 2024-04-22T21:40:05.236 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "", 2024-04-22T21:40:05.236 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:40:05.236 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": null, 2024-04-22T21:40:05.236 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "", 2024-04-22T21:40:05.236 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:40:05.236 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:40:05.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:05 smithi130 ceph-mon[82829]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:05.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:05 smithi130 ceph-mon[82829]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:05.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:05 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:40:05.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:05 smithi130 ceph-mon[85205]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:05.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:05 smithi130 ceph-mon[85205]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:05.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:05 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:40:05.842 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph health detail' 2024-04-22T21:40:06.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:05 smithi150 ceph-mon[65103]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:06.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:05 smithi150 ceph-mon[65103]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:06.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:05 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:40:07.904 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-04-22T21:40:08.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:08 smithi130 ceph-mon[85205]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:08.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:08 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/1288492943' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:40:08.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:08 smithi130 ceph-mon[82829]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:08.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:08 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/1288492943' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:40:08.583 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-04-22T21:40:08.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:08 smithi150 ceph-mon[65103]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:08.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:08 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/1288492943' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:40:09.141 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:40:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:40:08] "GET /metrics HTTP/1.1" 200 38117 "" "Prometheus/2.43.0" 2024-04-22T21:40:09.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:09 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:09.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:09 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:09.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:09 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:10.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:10 smithi130 ceph-mon[85205]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:10.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:10 smithi130 ceph-mon[82829]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:10.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:10 smithi150 ceph-mon[65103]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:11.529 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:11 smithi130 ceph-mon[82829]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:11.530 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:11 smithi130 ceph-mon[85205]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:11.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:11 smithi150 ceph-mon[65103]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:12.157 INFO:teuthology.orchestra.run.smithi130.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:40:12.480 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:12 smithi130 ceph-mon[82829]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:12.480 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:12 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:12.480 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:12 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:40:12.480 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:12 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:40:12.480 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:12 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:12.480 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:12 smithi130 ceph-mon[85205]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:12.480 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:12 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:12.480 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:12 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:40:12.481 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:12 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:40:12.481 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:12 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:12.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:12 smithi150 ceph-mon[65103]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:12.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:12 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:12.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:12 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:40:12.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:12 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:40:12.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:12 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:12.839 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T21:40:13.377 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:13 smithi130 ceph-mon[85205]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:40:13.377 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:13 smithi130 ceph-mon[85205]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:40:13.377 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:13 smithi130 ceph-mon[82829]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:40:13.377 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:13 smithi130 ceph-mon[82829]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:40:13.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:13 smithi150 ceph-mon[65103]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:40:13.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:13 smithi150 ceph-mon[65103]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:40:14.532 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[82829]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:14.532 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:14.532 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:40:14.532 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:14.532 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:14.532 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:14.532 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:14.532 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:14.532 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:14.532 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T21:40:14.533 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[85205]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:14.533 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:14.533 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:40:14.533 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:14.533 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:14.533 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:14.533 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:14.533 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:14.533 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:14.533 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:14 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T21:40:14.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:14 smithi150 ceph-mon[65103]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:14.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:14 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:14.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:14 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:40:14.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:14 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:14.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:14 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:14.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:14 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:14.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:14 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:14.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:14 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:14.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:14 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:14.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:14 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T21:40:15.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[85205]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:40:15.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[85205]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:40:15.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mgr 2024-04-22T21:40:15.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mon 2024-04-22T21:40:15.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all crash 2024-04-22T21:40:15.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[85205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T21:40:15.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[85205]: Upgrade: osd.3 is safe to restart 2024-04-22T21:40:15.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[82829]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:40:15.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[82829]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:40:15.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mgr 2024-04-22T21:40:15.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mon 2024-04-22T21:40:15.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all crash 2024-04-22T21:40:15.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[82829]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T21:40:15.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:15 smithi130 ceph-mon[82829]: Upgrade: osd.3 is safe to restart 2024-04-22T21:40:15.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:15 smithi150 ceph-mon[65103]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:40:15.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:15 smithi150 ceph-mon[65103]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:40:15.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:15 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mgr 2024-04-22T21:40:15.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:15 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mon 2024-04-22T21:40:15.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:15 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all crash 2024-04-22T21:40:15.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:15 smithi150 ceph-mon[65103]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T21:40:15.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:15 smithi150 ceph-mon[65103]: Upgrade: osd.3 is safe to restart 2024-04-22T21:40:16.502 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:16 smithi130 ceph-mon[82829]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:16.503 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:16 smithi130 ceph-mon[85205]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:16.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:16 smithi150 ceph-mon[65103]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:17.020 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:40:17.701 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:40:17.701 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (6m) 42s ago 28m 27.7M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:40:17.701 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (6m) 4m ago 28m 83.7M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:40:17.701 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (6m) 42s ago 28m 60.5M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:40:17.701 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (8m) 4m ago 33m 448M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:40:17.701 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (17m) 42s ago 35m 508M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:40:17.701 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (4m) 42s ago 35m 55.8M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:40:17.701 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (5m) 4m ago 33m 38.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:40:17.701 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (3m) 42s ago 34m 39.5M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:40:17.701 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (7m) 42s ago 29m 15.6M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:40:17.702 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (7m) 4m ago 29m 15.8M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:40:17.702 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (2m) 42s ago 33m 46.2M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:40:17.702 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (100s) 42s ago 32m 52.3M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:40:17.702 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (47s) 42s ago 32m 12.7M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 027e1ac8c177 2024-04-22T21:40:17.702 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (32m) 42s ago 32m 56.1M 2992M 17.2.0 e1d6a67b021e 98f5f837f66f 2024-04-22T21:40:17.702 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (31m) 4m ago 31m 55.5M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:40:17.702 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (31m) 4m ago 31m 52.8M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:40:17.702 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (30m) 4m ago 30m 54.5M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:40:17.702 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (30m) 4m ago 30m 53.8M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:40:17.702 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (7m) 4m ago 29m 69.5M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:40:17.702 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (28m) 42s ago 28m 79.9M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:40:17.702 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (28m) 4m ago 28m 76.2M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:40:17.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:17 smithi130 ceph-mon[85205]: Upgrade: Updating osd.3 2024-04-22T21:40:17.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:17.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T21:40:17.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:40:17.703 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:17 smithi130 ceph-mon[85205]: Deploying daemon osd.3 on smithi130 2024-04-22T21:40:17.703 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:17 smithi130 ceph-mon[82829]: Upgrade: Updating osd.3 2024-04-22T21:40:17.703 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:17.703 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T21:40:17.703 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:40:17.703 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:17 smithi130 ceph-mon[82829]: Deploying daemon osd.3 on smithi130 2024-04-22T21:40:17.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:17 smithi150 ceph-mon[65103]: Upgrade: Updating osd.3 2024-04-22T21:40:17.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:17.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T21:40:17.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:40:17.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:17 smithi150 ceph-mon[65103]: Deploying daemon osd.3 on smithi130 2024-04-22T21:40:18.102 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:40:18.102 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:40:18.102 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 8 2024-04-22T21:40:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:40:18.104 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:40:18.457 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:40:18.457 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:40:18.457 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:40:18.457 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-22T21:40:18.457 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:40:18.457 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "3/8 daemons upgraded", 2024-04-22T21:40:18.457 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T21:40:18.457 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:40:18.457 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:40:18.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:18 smithi130 ceph-mon[82829]: from='client.64372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:18.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:18 smithi130 ceph-mon[82829]: from='client.64378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:18.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:18 smithi130 ceph-mon[82829]: from='client.64384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:18.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:18 smithi130 ceph-mon[82829]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:18.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:18 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/1546679463' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:18.672 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:18 smithi130 ceph-mon[85205]: from='client.64372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:18.673 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:18 smithi130 ceph-mon[85205]: from='client.64378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:18.673 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:18 smithi130 ceph-mon[85205]: from='client.64384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:18.673 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:18 smithi130 ceph-mon[85205]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:18.673 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:18 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/1546679463' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:18.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:18 smithi150 ceph-mon[65103]: from='client.64372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:18.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:18 smithi150 ceph-mon[65103]: from='client.64378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:18.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:18 smithi150 ceph-mon[65103]: from='client.64384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:18.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:18 smithi150 ceph-mon[65103]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:18.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:18 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/1546679463' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:19.038 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:40:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:40:18] "GET /metrics HTTP/1.1" 200 38117 "" "Prometheus/2.43.0" 2024-04-22T21:40:19.660 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:19 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:19.660 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:19 smithi130 ceph-mon[82829]: from='client.54462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:19.660 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:19 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:19.660 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:19 smithi130 ceph-mon[85205]: from='client.54462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:19.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:19 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:19.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:19 smithi150 ceph-mon[65103]: from='client.54462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:20.274 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:20 smithi130 systemd[1]: Stopping Ceph osd.3 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:40:20.544 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:20 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3[38170]: 2024-04-22T21:40:20.291+0000 7fd73498a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:40:20.544 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:20 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3[38170]: 2024-04-22T21:40:20.291+0000 7fd73498a700 -1 osd.3 128 *** Got signal Terminated *** 2024-04-22T21:40:20.545 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:20 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3[38170]: 2024-04-22T21:40:20.291+0000 7fd73498a700 -1 osd.3 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:40:20.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:20 smithi150 ceph-mon[65103]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:20.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:20 smithi150 ceph-mon[65103]: osd.3 marked itself down and dead 2024-04-22T21:40:20.987 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:20 smithi130 ceph-mon[82829]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:20.987 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:20 smithi130 ceph-mon[82829]: osd.3 marked itself down and dead 2024-04-22T21:40:20.987 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:20 smithi130 ceph-mon[85205]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:20.987 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:20 smithi130 ceph-mon[85205]: osd.3 marked itself down and dead 2024-04-22T21:40:21.256 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:21 smithi130 podman[109837]: 2024-04-22 21:40:21.006367608 +0000 UTC m=+0.830634476 container died 98f5f837f66f715a1cfa95bfc0fa59c55983d8912c7f977e562e161dcc71a4f1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3, io.openshift.expose-services=, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, release=754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, vcs-type=git, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , version=8, com.redhat.component=centos-stream-container, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870) 2024-04-22T21:40:21.545 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:21 smithi130 podman[109837]: 2024-04-22 21:40:21.374912455 +0000 UTC m=+1.199179328 container cleanup 98f5f837f66f715a1cfa95bfc0fa59c55983d8912c7f977e562e161dcc71a4f1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_CLEAN=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.openshift.expose-services=, version=8, io.openshift.tags=base centos centos-stream) 2024-04-22T21:40:21.545 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:21 smithi130 bash[109837]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3 2024-04-22T21:40:21.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:21 smithi150 ceph-mon[65103]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:40:21.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:21 smithi150 ceph-mon[65103]: osdmap e129: 8 total, 7 up, 8 in 2024-04-22T21:40:21.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:21.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:40:21.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:21 smithi130 ceph-mon[82829]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:40:21.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:21 smithi130 ceph-mon[82829]: osdmap e129: 8 total, 7 up, 8 in 2024-04-22T21:40:21.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:21.973 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:40:21.973 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:21 smithi130 ceph-mon[85205]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:40:21.973 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:21 smithi130 ceph-mon[85205]: osdmap e129: 8 total, 7 up, 8 in 2024-04-22T21:40:21.973 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:21.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:40:21.974 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:21 smithi130 podman[109854]: 2024-04-22 21:40:21.651183693 +0000 UTC m=+0.642442394 container remove 98f5f837f66f715a1cfa95bfc0fa59c55983d8912c7f977e562e161dcc71a4f1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.openshift.expose-services=, distribution-scope=public, release=754, ceph=True, vcs-type=git, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., architecture=x86_64) 2024-04-22T21:40:22.260 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:21 smithi130 podman[109948]: 2024-04-22 21:40:21.89516109 +0000 UTC m=+0.018153577 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:40:22.260 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:22 smithi130 podman[109948]: 2024-04-22 21:40:22.021893837 +0000 UTC m=+0.144886333 container create 2875ffca35c8ca3c81cb4544bb0d4525bf0672a9d7d54f946a70e3c2d9736e02 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-22T21:40:22.545 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:22 smithi130 podman[109948]: 2024-04-22 21:40:22.279789734 +0000 UTC m=+0.402782205 container init 2875ffca35c8ca3c81cb4544bb0d4525bf0672a9d7d54f946a70e3c2d9736e02 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-22T21:40:22.545 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:22 smithi130 podman[109948]: 2024-04-22 21:40:22.283194261 +0000 UTC m=+0.406186721 container start 2875ffca35c8ca3c81cb4544bb0d4525bf0672a9d7d54f946a70e3c2d9736e02 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-56e81df, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-22T21:40:22.545 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:22 smithi130 podman[109948]: 2024-04-22 21:40:22.363115151 +0000 UTC m=+0.486107614 container attach 2875ffca35c8ca3c81cb4544bb0d4525bf0672a9d7d54f946a70e3c2d9736e02 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.build-date=20240416, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-22T21:40:22.847 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:22 smithi130 podman[109948]: 2024-04-22 21:40:22.586558546 +0000 UTC m=+0.709551002 container died 2875ffca35c8ca3c81cb4544bb0d4525bf0672a9d7d54f946a70e3c2d9736e02 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-deactivate, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-56e81df, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-22T21:40:22.848 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:22 smithi130 ceph-mon[82829]: osdmap e130: 8 total, 7 up, 8 in 2024-04-22T21:40:22.848 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:22 smithi130 ceph-mon[82829]: pgmap v148: 161 pgs: 21 active+undersized+degraded, 7 stale+active+clean, 25 active+undersized, 108 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 58/627 objects degraded (9.250%) 2024-04-22T21:40:22.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:22 smithi130 ceph-mon[85205]: osdmap e130: 8 total, 7 up, 8 in 2024-04-22T21:40:22.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:22 smithi130 ceph-mon[85205]: pgmap v148: 161 pgs: 21 active+undersized+degraded, 7 stale+active+clean, 25 active+undersized, 108 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 58/627 objects degraded (9.250%) 2024-04-22T21:40:22.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:22 smithi150 ceph-mon[65103]: osdmap e130: 8 total, 7 up, 8 in 2024-04-22T21:40:22.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:22 smithi150 ceph-mon[65103]: pgmap v148: 161 pgs: 21 active+undersized+degraded, 7 stale+active+clean, 25 active+undersized, 108 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 58/627 objects degraded (9.250%) 2024-04-22T21:40:23.100 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:23 smithi130 podman[109975]: 2024-04-22 21:40:23.116460416 +0000 UTC m=+0.523937405 container remove 2875ffca35c8ca3c81cb4544bb0d4525bf0672a9d7d54f946a70e3c2d9736e02 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=main-56e81df, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:40:23.536 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:23 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-22T21:40:23.536 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:23 smithi130 systemd[1]: Stopped Ceph osd.3 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:40:23.536 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:23 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.3.service: Consumed 16.992s CPU time. 2024-04-22T21:40:23.536 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:23 smithi130 systemd[1]: Starting Ceph osd.3 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:40:23.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:23 smithi130 ceph-mon[85205]: Health check failed: Degraded data redundancy: 58/627 objects degraded (9.250%), 21 pgs degraded (PG_DEGRADED) 2024-04-22T21:40:23.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:23 smithi130 ceph-mon[82829]: Health check failed: Degraded data redundancy: 58/627 objects degraded (9.250%), 21 pgs degraded (PG_DEGRADED) 2024-04-22T21:40:23.795 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:23 smithi130 podman[110080]: 2024-04-22 21:40:23.458405121 +0000 UTC m=+0.017985577 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:40:23.795 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:23 smithi130 podman[110080]: 2024-04-22 21:40:23.627044144 +0000 UTC m=+0.186624591 container create fa79679f2d877d4d2ab922e6d9cf4f5055702d04e37135ca9af60345e1873a49 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-22T21:40:23.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:23 smithi150 ceph-mon[65103]: Health check failed: Degraded data redundancy: 58/627 objects degraded (9.250%), 21 pgs degraded (PG_DEGRADED) 2024-04-22T21:40:24.294 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:23 smithi130 podman[110080]: 2024-04-22 21:40:23.89522672 +0000 UTC m=+0.454807164 container init fa79679f2d877d4d2ab922e6d9cf4f5055702d04e37135ca9af60345e1873a49 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-56e81df, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T21:40:24.295 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:23 smithi130 podman[110080]: 2024-04-22 21:40:23.898840693 +0000 UTC m=+0.458421137 container start fa79679f2d877d4d2ab922e6d9cf4f5055702d04e37135ca9af60345e1873a49 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-56e81df, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-22T21:40:24.295 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:23 smithi130 podman[110080]: 2024-04-22 21:40:23.978441056 +0000 UTC m=+0.538021518 container attach fa79679f2d877d4d2ab922e6d9cf4f5055702d04e37135ca9af60345e1873a49 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, RELEASE=main-56e81df, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-22T21:40:24.627 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:24 smithi130 ceph-mon[82829]: pgmap v149: 161 pgs: 26 active+undersized+degraded, 1 stale+active+clean, 34 active+undersized, 100 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 80/627 objects degraded (12.759%) 2024-04-22T21:40:24.627 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:24 smithi130 ceph-mon[85205]: pgmap v149: 161 pgs: 26 active+undersized+degraded, 1 stale+active+clean, 34 active+undersized, 100 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 80/627 objects degraded (12.759%) 2024-04-22T21:40:24.628 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate[110095]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:40:24.628 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate[110095]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:40:24.628 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 bash[110080]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:40:24.628 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 bash[110080]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:40:24.628 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 bash[110080]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T21:40:24.628 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 bash[110080]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T21:40:24.628 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 bash[110080]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T21:40:24.628 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 bash[110080]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T21:40:24.628 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 bash[110080]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-22T21:40:24.628 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 bash[110080]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T21:40:24.629 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 bash[110080]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-22T21:40:24.629 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate[110095]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T21:40:24.629 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate[110095]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T21:40:24.629 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate[110095]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T21:40:24.629 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate[110095]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T21:40:24.629 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate[110095]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-22T21:40:24.629 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate[110095]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T21:40:24.629 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate[110095]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-22T21:40:24.629 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 podman[110080]: 2024-04-22 21:40:24.398222046 +0000 UTC m=+0.957802494 container died fa79679f2d877d4d2ab922e6d9cf4f5055702d04e37135ca9af60345e1873a49 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD) 2024-04-22T21:40:24.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:24 smithi150 ceph-mon[65103]: pgmap v149: 161 pgs: 26 active+undersized+degraded, 1 stale+active+clean, 34 active+undersized, 100 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 80/627 objects degraded (12.759%) 2024-04-22T21:40:25.287 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:24 smithi130 podman[110232]: 2024-04-22 21:40:24.965735301 +0000 UTC m=+0.560580618 container remove fa79679f2d877d4d2ab922e6d9cf4f5055702d04e37135ca9af60345e1873a49 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.build-date=20240416, RELEASE=main-56e81df, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:40:25.288 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:25 smithi130 podman[110304]: 2024-04-22 21:40:25.095854798 +0000 UTC m=+0.020346189 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:40:25.288 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:25 smithi130 podman[110304]: 2024-04-22 21:40:25.199965236 +0000 UTC m=+0.124456639 container create ca6c16e44590fd32921fad575a182be35ab58b297cd5a96677559e9f3736f3ab (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, RELEASE=main-56e81df, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T21:40:25.794 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:25 smithi130 podman[110304]: 2024-04-22 21:40:25.443871635 +0000 UTC m=+0.368363008 container init ca6c16e44590fd32921fad575a182be35ab58b297cd5a96677559e9f3736f3ab (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-22T21:40:25.795 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:25 smithi130 podman[110304]: 2024-04-22 21:40:25.446953273 +0000 UTC m=+0.371444647 container start ca6c16e44590fd32921fad575a182be35ab58b297cd5a96677559e9f3736f3ab (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, RELEASE=main-56e81df) 2024-04-22T21:40:25.795 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:25 smithi130 ceph-osd[110322]: -- 172.21.15.130:0/3936323009 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a8eceb7680 con 0x55a8ece81400 2024-04-22T21:40:25.795 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:25 smithi130 bash[110304]: ca6c16e44590fd32921fad575a182be35ab58b297cd5a96677559e9f3736f3ab 2024-04-22T21:40:25.795 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:25 smithi130 systemd[1]: Started Ceph osd.3 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:40:26.294 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3[110318]: 2024-04-22T21:40:25.996+0000 7f5a0ff0c7c0 -1 Falling back to public interface 2024-04-22T21:40:26.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:26 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:26.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:26 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:26.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:26 smithi130 ceph-mon[82829]: pgmap v150: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 80/627 objects degraded (12.759%) 2024-04-22T21:40:26.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:26 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:26.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:26 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:26.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:26 smithi130 ceph-mon[85205]: pgmap v150: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 80/627 objects degraded (12.759%) 2024-04-22T21:40:26.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:26 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:26.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:26 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:26.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:26 smithi150 ceph-mon[65103]: pgmap v150: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 80/627 objects degraded (12.759%) 2024-04-22T21:40:28.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:27 smithi130 ceph-mon[85205]: pgmap v151: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 80/627 objects degraded (12.759%) 2024-04-22T21:40:28.034 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:27 smithi130 ceph-mon[82829]: pgmap v151: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 80/627 objects degraded (12.759%) 2024-04-22T21:40:28.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:27 smithi150 ceph-mon[65103]: pgmap v151: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 80/627 objects degraded (12.759%) 2024-04-22T21:40:29.044 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:40:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:40:28] "GET /metrics HTTP/1.1" 200 38115 "" "Prometheus/2.43.0" 2024-04-22T21:40:29.045 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3[110318]: 2024-04-22T21:40:28.710+0000 7f5a0ff0c7c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-04-22T21:40:29.045 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3[110318]: 2024-04-22T21:40:28.740+0000 7f5a0ff0c7c0 -1 osd.3 128 log_to_monitors true 2024-04-22T21:40:29.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:28 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:29.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:28 smithi130 ceph-mon[82829]: from='osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T21:40:29.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:28 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:29.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:28 smithi130 ceph-mon[85205]: from='osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T21:40:29.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:28 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:29.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:28 smithi150 ceph-mon[65103]: from='osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T21:40:29.513 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:40:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3[110318]: 2024-04-22T21:40:29.249+0000 7f59fd35c700 -1 osd.3 128 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:40:30.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:29 smithi130 ceph-mon[82829]: from='osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T21:40:30.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:29 smithi130 ceph-mon[82829]: osdmap e131: 8 total, 7 up, 8 in 2024-04-22T21:40:30.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:29 smithi130 ceph-mon[82829]: from='osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:40:30.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:29 smithi130 ceph-mon[82829]: from='osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127]' entity='osd.3' 2024-04-22T21:40:30.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:29 smithi130 ceph-mon[82829]: pgmap v153: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 80/627 objects degraded (12.759%) 2024-04-22T21:40:30.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:29 smithi130 ceph-mon[85205]: from='osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T21:40:30.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:29 smithi130 ceph-mon[85205]: osdmap e131: 8 total, 7 up, 8 in 2024-04-22T21:40:30.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:29 smithi130 ceph-mon[85205]: from='osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:40:30.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:29 smithi130 ceph-mon[85205]: from='osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127]' entity='osd.3' 2024-04-22T21:40:30.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:29 smithi130 ceph-mon[85205]: pgmap v153: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 80/627 objects degraded (12.759%) 2024-04-22T21:40:30.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:29 smithi150 ceph-mon[65103]: from='osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T21:40:30.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:29 smithi150 ceph-mon[65103]: osdmap e131: 8 total, 7 up, 8 in 2024-04-22T21:40:30.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:29 smithi150 ceph-mon[65103]: from='osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-04-22T21:40:30.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:29 smithi150 ceph-mon[65103]: from='osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127]' entity='osd.3' 2024-04-22T21:40:30.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:29 smithi150 ceph-mon[65103]: pgmap v153: 161 pgs: 26 active+undersized+degraded, 40 active+undersized, 95 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 80/627 objects degraded (12.759%) 2024-04-22T21:40:31.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:31 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:31.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:31 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:31.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:31 smithi130 ceph-mon[85205]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:40:31.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:31 smithi130 ceph-mon[85205]: osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127] boot 2024-04-22T21:40:31.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:31 smithi130 ceph-mon[85205]: osdmap e132: 8 total, 8 up, 8 in 2024-04-22T21:40:31.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:31 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:40:31.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:31 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:31.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:31 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:31.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:31 smithi130 ceph-mon[82829]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:40:31.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:31 smithi130 ceph-mon[82829]: osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127] boot 2024-04-22T21:40:31.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:31 smithi130 ceph-mon[82829]: osdmap e132: 8 total, 8 up, 8 in 2024-04-22T21:40:31.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:31 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:40:31.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:31 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:31.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:31 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:31.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:31 smithi150 ceph-mon[65103]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:40:31.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:31 smithi150 ceph-mon[65103]: osd.3 [v2:172.21.15.130:6826/359723127,v1:172.21.15.130:6827/359723127] boot 2024-04-22T21:40:31.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:31 smithi150 ceph-mon[65103]: osdmap e132: 8 total, 8 up, 8 in 2024-04-22T21:40:31.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:31 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T21:40:32.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:32.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:32.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:32 smithi130 ceph-mon[85205]: osdmap e133: 8 total, 8 up, 8 in 2024-04-22T21:40:32.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:32 smithi130 ceph-mon[85205]: pgmap v156: 161 pgs: 14 active+undersized+degraded, 26 active+undersized, 121 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 49/627 objects degraded (7.815%) 2024-04-22T21:40:32.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:32.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:32.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:32 smithi130 ceph-mon[82829]: osdmap e133: 8 total, 8 up, 8 in 2024-04-22T21:40:32.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:32 smithi130 ceph-mon[82829]: pgmap v156: 161 pgs: 14 active+undersized+degraded, 26 active+undersized, 121 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 49/627 objects degraded (7.815%) 2024-04-22T21:40:32.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:32.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:32.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:32 smithi150 ceph-mon[65103]: osdmap e133: 8 total, 8 up, 8 in 2024-04-22T21:40:32.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:32 smithi150 ceph-mon[65103]: pgmap v156: 161 pgs: 14 active+undersized+degraded, 26 active+undersized, 121 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 49/627 objects degraded (7.815%) 2024-04-22T21:40:33.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:33 smithi130 ceph-mon[82829]: Health check update: Degraded data redundancy: 80/627 objects degraded (12.759%), 26 pgs degraded (PG_DEGRADED) 2024-04-22T21:40:33.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:33 smithi130 ceph-mon[85205]: Health check update: Degraded data redundancy: 80/627 objects degraded (12.759%), 26 pgs degraded (PG_DEGRADED) 2024-04-22T21:40:33.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:33 smithi150 ceph-mon[65103]: Health check update: Degraded data redundancy: 80/627 objects degraded (12.759%), 26 pgs degraded (PG_DEGRADED) 2024-04-22T21:40:34.256 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:34 smithi130 ceph-mon[82829]: pgmap v157: 161 pgs: 11 active+undersized+degraded, 21 active+undersized, 129 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 35/627 objects degraded (5.582%) 2024-04-22T21:40:34.256 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:34 smithi130 ceph-mon[85205]: pgmap v157: 161 pgs: 11 active+undersized+degraded, 21 active+undersized, 129 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 35/627 objects degraded (5.582%) 2024-04-22T21:40:34.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:34 smithi150 ceph-mon[65103]: pgmap v157: 161 pgs: 11 active+undersized+degraded, 21 active+undersized, 129 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 35/627 objects degraded (5.582%) 2024-04-22T21:40:35.043 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:40:34 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:40:34.866+0000 7fd7212f0700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-22T21:40:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:40:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:40:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:40:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T21:40:36.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T21:40:36.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-22T21:40:36.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:40:36.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:36.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:40:36.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:36.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:36.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:40:36.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:40:36.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:36.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:40:36.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:36.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:36.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:36.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T21:40:36.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T21:40:36.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-22T21:40:36.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:40:36.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:36.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:40:36.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:36.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:36.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:40:36.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:40:36.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:36.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:40:36.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:36.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:36.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:36.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T21:40:36.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T21:40:36.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-22T21:40:36.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:40:36.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:36.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:40:37.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:36 smithi130 ceph-mon[82829]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/627 objects degraded (5.582%), 11 pgs degraded) 2024-04-22T21:40:37.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:36 smithi130 ceph-mon[82829]: Cluster is now healthy 2024-04-22T21:40:37.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:36 smithi130 ceph-mon[85205]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/627 objects degraded (5.582%), 11 pgs degraded) 2024-04-22T21:40:37.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:36 smithi130 ceph-mon[85205]: Cluster is now healthy 2024-04-22T21:40:37.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:36 smithi150 ceph-mon[65103]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/627 objects degraded (5.582%), 11 pgs degraded) 2024-04-22T21:40:37.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:36 smithi150 ceph-mon[65103]: Cluster is now healthy 2024-04-22T21:40:38.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:37 smithi130 ceph-mon[85205]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:40:38.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:37 smithi130 ceph-mon[82829]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:40:38.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:37 smithi150 ceph-mon[65103]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:40:39.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:38 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:38 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:39.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:40:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:40:38] "GET /metrics HTTP/1.1" 200 38128 "" "Prometheus/2.43.0" 2024-04-22T21:40:39.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:38 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:39 smithi130 ceph-mon[85205]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 216 B/s rd, 0 op/s 2024-04-22T21:40:40.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:39 smithi130 ceph-mon[82829]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 216 B/s rd, 0 op/s 2024-04-22T21:40:40.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:39 smithi150 ceph-mon[65103]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 216 B/s rd, 0 op/s 2024-04-22T21:40:42.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:42 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:42.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:42 smithi130 ceph-mon[82829]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 195 B/s rd, 0 op/s 2024-04-22T21:40:42.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:42 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:42.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:42 smithi130 ceph-mon[85205]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 195 B/s rd, 0 op/s 2024-04-22T21:40:42.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:42 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:42.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:42 smithi150 ceph-mon[65103]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 195 B/s rd, 0 op/s 2024-04-22T21:40:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:43 smithi130 ceph-mon[85205]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:40:44.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:43 smithi130 ceph-mon[82829]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:40:44.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:43 smithi150 ceph-mon[65103]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T21:40:46.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:45 smithi130 ceph-mon[82829]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:40:46.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:45 smithi130 ceph-mon[85205]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:40:46.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:45 smithi150 ceph-mon[65103]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T21:40:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:47 smithi130 ceph-mon[85205]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:48.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:47 smithi130 ceph-mon[82829]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:48.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:47 smithi150 ceph-mon[65103]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:48.823 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:40:49.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:48 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:49.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:40:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:40:48] "GET /metrics HTTP/1.1" 200 38128 "" "Prometheus/2.43.0" 2024-04-22T21:40:49.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:48 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:49.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:48 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:49.478 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:40:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (7m) 19s ago 29m 27.7M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:40:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (6m) 4m ago 29m 83.7M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:40:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (6m) 19s ago 28m 60.5M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:40:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (8m) 4m ago 34m 448M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:40:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (18m) 19s ago 35m 510M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:40:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (4m) 19s ago 35m 55.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:40:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (5m) 4m ago 34m 38.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:40:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (4m) 19s ago 34m 41.2M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:40:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (8m) 19s ago 30m 15.7M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:40:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (8m) 4m ago 30m 15.8M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:40:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (2m) 19s ago 33m 47.6M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:40:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (2m) 19s ago 33m 52.4M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:40:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (78s) 19s ago 33m 41.5M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 027e1ac8c177 2024-04-22T21:40:49.480 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (24s) 19s ago 32m 12.1M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 ca6c16e44590 2024-04-22T21:40:49.480 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (32m) 4m ago 32m 55.5M 3248M 17.2.0 e1d6a67b021e d74dae492d11 2024-04-22T21:40:49.480 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (31m) 4m ago 31m 52.8M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:40:49.480 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (31m) 4m ago 31m 54.5M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:40:49.480 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (31m) 4m ago 31m 53.8M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:40:49.480 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (7m) 4m ago 30m 69.5M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:40:49.480 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (28m) 19s ago 28m 80.2M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:40:49.480 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (28m) 4m ago 28m 76.2M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:40:49.880 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:40:49.880 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:40:49.880 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:40:49.880 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:40:49.880 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:40:49.880 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:40:49.880 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:40:49.880 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:40:49.880 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-22T21:40:49.881 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 4 2024-04-22T21:40:49.881 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:40:49.881 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:40:49.881 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:40:49.881 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:40:49.881 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:40:49.881 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-22T21:40:49.881 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 9 2024-04-22T21:40:49.881 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:40:49.881 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:40:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:49 smithi130 ceph-mon[85205]: from='client.54474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:49 smithi130 ceph-mon[85205]: from='client.54480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:49 smithi130 ceph-mon[85205]: from='client.54486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:49 smithi130 ceph-mon[85205]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:50.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:49 smithi130 ceph-mon[82829]: from='client.54474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:50.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:49 smithi130 ceph-mon[82829]: from='client.54480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:50.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:49 smithi130 ceph-mon[82829]: from='client.54486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:50.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:49 smithi130 ceph-mon[82829]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:50.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:49 smithi150 ceph-mon[65103]: from='client.54474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:50.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:49 smithi150 ceph-mon[65103]: from='client.54480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:50.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:49 smithi150 ceph-mon[65103]: from='client.54486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:50.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:49 smithi150 ceph-mon[65103]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:40:50.220 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:40:50.220 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:40:50.220 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:40:50.221 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-22T21:40:50.221 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:40:50.221 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "4/8 daemons upgraded", 2024-04-22T21:40:50.221 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T21:40:50.221 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:40:50.221 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:40:51.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T21:40:51.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[82829]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T21:40:51.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[82829]: Upgrade: osd.4 is safe to restart 2024-04-22T21:40:51.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/2479621588' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:51.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[82829]: from='client.64432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:51.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:51.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:40:51.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T21:40:51.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[85205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T21:40:51.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[85205]: Upgrade: osd.4 is safe to restart 2024-04-22T21:40:51.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/2479621588' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:51.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[85205]: from='client.64432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:51.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:51.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:50 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:40:51.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:50 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T21:40:51.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:50 smithi150 ceph-mon[65103]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T21:40:51.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:50 smithi150 ceph-mon[65103]: Upgrade: osd.4 is safe to restart 2024-04-22T21:40:51.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:50 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/2479621588' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:40:51.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:50 smithi150 ceph-mon[65103]: from='client.64432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:40:51.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:50 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:51.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:50 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:40:52.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:52 smithi150 ceph-mon[65103]: Upgrade: Updating osd.4 2024-04-22T21:40:52.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:52.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T21:40:52.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:52 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:40:52.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:52 smithi150 ceph-mon[65103]: Deploying daemon osd.4 on smithi150 2024-04-22T21:40:52.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:52 smithi150 ceph-mon[65103]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:53.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:52 smithi130 ceph-mon[85205]: Upgrade: Updating osd.4 2024-04-22T21:40:53.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:53.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T21:40:53.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:52 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:40:53.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:52 smithi130 ceph-mon[85205]: Deploying daemon osd.4 on smithi150 2024-04-22T21:40:53.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:52 smithi130 ceph-mon[85205]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:53.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:52 smithi130 ceph-mon[82829]: Upgrade: Updating osd.4 2024-04-22T21:40:53.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:40:53.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T21:40:53.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:52 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:40:53.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:52 smithi130 ceph-mon[82829]: Deploying daemon osd.4 on smithi150 2024-04-22T21:40:53.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:52 smithi130 ceph-mon[82829]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:53.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:53 smithi150 ceph-mon[65103]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:53 smithi130 ceph-mon[85205]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:54.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:53 smithi130 ceph-mon[82829]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:40:55.138 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:54 smithi150 systemd[1]: Stopping Ceph osd.4 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:40:55.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:55 smithi130 ceph-mon[85205]: osd.4 marked itself down and dead 2024-04-22T21:40:55.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:55 smithi130 ceph-mon[82829]: osd.4 marked itself down and dead 2024-04-22T21:40:55.537 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:55 smithi150 ceph-mon[65103]: osd.4 marked itself down and dead 2024-04-22T21:40:55.537 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:55 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[26102]: 2024-04-22T21:40:55.155+0000 7fad2224f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:40:55.537 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:55 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[26102]: 2024-04-22T21:40:55.155+0000 7fad2224f700 -1 osd.4 133 *** Got signal Terminated *** 2024-04-22T21:40:55.537 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:55 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[26102]: 2024-04-22T21:40:55.155+0000 7fad2224f700 -1 osd.4 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:40:55.838 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:55 smithi150 podman[75236]: 2024-04-22 21:40:55.556038342 +0000 UTC m=+0.508472852 container died d74dae492d110d98597aa61281f5fcc3e0925e14c5c6f590ac3fb453a771cb1c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, vcs-type=git, name=centos-stream, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., release=754, build-date=2022-05-03T08:36:31.336870) 2024-04-22T21:40:56.098 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:55 smithi150 podman[75236]: 2024-04-22 21:40:55.94000701 +0000 UTC m=+0.892441520 container cleanup d74dae492d110d98597aa61281f5fcc3e0925e14c5c6f590ac3fb453a771cb1c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, GIT_CLEAN=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, CEPH_POINT_RELEASE=-17.2.0, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, architecture=x86_64, com.redhat.component=centos-stream-container, vcs-type=git) 2024-04-22T21:40:56.099 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:55 smithi150 bash[75236]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4 2024-04-22T21:40:56.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:56 smithi130 ceph-mon[85205]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:40:56.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:56 smithi130 ceph-mon[85205]: osdmap e134: 8 total, 7 up, 8 in 2024-04-22T21:40:56.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:56 smithi130 ceph-mon[85205]: pgmap v169: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:40:56.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:56 smithi130 ceph-mon[85205]: osdmap e135: 8 total, 7 up, 8 in 2024-04-22T21:40:56.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:56 smithi130 ceph-mon[82829]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:40:56.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:56 smithi130 ceph-mon[82829]: osdmap e134: 8 total, 7 up, 8 in 2024-04-22T21:40:56.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:56 smithi130 ceph-mon[82829]: pgmap v169: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:40:56.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:56 smithi130 ceph-mon[82829]: osdmap e135: 8 total, 7 up, 8 in 2024-04-22T21:40:56.556 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:56 smithi150 ceph-mon[65103]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:40:56.557 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:56 smithi150 ceph-mon[65103]: osdmap e134: 8 total, 7 up, 8 in 2024-04-22T21:40:56.557 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:56 smithi150 ceph-mon[65103]: pgmap v169: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:40:56.557 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:56 smithi150 ceph-mon[65103]: osdmap e135: 8 total, 7 up, 8 in 2024-04-22T21:40:56.557 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:56 smithi150 podman[75263]: 2024-04-22 21:40:56.24992452 +0000 UTC m=+0.694265760 container remove d74dae492d110d98597aa61281f5fcc3e0925e14c5c6f590ac3fb453a771cb1c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, vendor=Red Hat, Inc., GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git) 2024-04-22T21:40:56.809 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:56 smithi150 podman[75360]: 2024-04-22 21:40:56.575928182 +0000 UTC m=+0.109092924 container create a9e4f7b5de747626d8869e6f7b4806047afde8ff5e9d55991753f54092b2139c (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-56e81df, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-22T21:40:56.809 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:56 smithi150 podman[75360]: 2024-04-22 21:40:56.485222497 +0000 UTC m=+0.018387197 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:40:57.099 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:56 smithi150 podman[75360]: 2024-04-22 21:40:56.828641113 +0000 UTC m=+0.361805822 container init a9e4f7b5de747626d8869e6f7b4806047afde8ff5e9d55991753f54092b2139c (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-22T21:40:57.099 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:56 smithi150 podman[75360]: 2024-04-22 21:40:56.83220554 +0000 UTC m=+0.365370250 container start a9e4f7b5de747626d8869e6f7b4806047afde8ff5e9d55991753f54092b2139c (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-deactivate, RELEASE=main-56e81df, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS) 2024-04-22T21:40:57.099 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:56 smithi150 podman[75360]: 2024-04-22 21:40:56.920396061 +0000 UTC m=+0.453560763 container attach a9e4f7b5de747626d8869e6f7b4806047afde8ff5e9d55991753f54092b2139c (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-deactivate, RELEASE=main-56e81df, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:40:57.471 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:57 smithi150 podman[75360]: 2024-04-22 21:40:57.127827529 +0000 UTC m=+0.660992230 container died a9e4f7b5de747626d8869e6f7b4806047afde8ff5e9d55991753f54092b2139c (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-22T21:40:58.098 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:57 smithi150 podman[75387]: 2024-04-22 21:40:57.775988699 +0000 UTC m=+0.638436492 container remove a9e4f7b5de747626d8869e6f7b4806047afde8ff5e9d55991753f54092b2139c (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-deactivate, RELEASE=main-56e81df, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-22T21:40:58.099 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:57 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-22T21:40:58.099 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:57 smithi150 systemd[1]: Stopped Ceph osd.4 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:40:58.099 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:57 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.4.service: Consumed 16.722s CPU time. 2024-04-22T21:40:58.099 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:57 smithi150 systemd[1]: Starting Ceph osd.4 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:40:58.459 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:58 smithi150 ceph-mon[65103]: pgmap v171: 161 pgs: 24 active+undersized+degraded, 9 stale+active+clean, 28 active+undersized, 100 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 98/627 objects degraded (15.630%) 2024-04-22T21:40:58.459 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 podman[75491]: 2024-04-22 21:40:58.116772543 +0000 UTC m=+0.018493178 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:40:58.459 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 podman[75491]: 2024-04-22 21:40:58.225909407 +0000 UTC m=+0.127630088 container create e1a204f1394bbabb3adfa32b40f525da7fe7742532dc3e3e0381129245d4c3d7 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate, RELEASE=main-56e81df, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-22T21:40:58.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:58 smithi130 ceph-mon[85205]: pgmap v171: 161 pgs: 24 active+undersized+degraded, 9 stale+active+clean, 28 active+undersized, 100 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 98/627 objects degraded (15.630%) 2024-04-22T21:40:58.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:58 smithi130 ceph-mon[82829]: pgmap v171: 161 pgs: 24 active+undersized+degraded, 9 stale+active+clean, 28 active+undersized, 100 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 98/627 objects degraded (15.630%) 2024-04-22T21:40:58.848 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 podman[75491]: 2024-04-22 21:40:58.478367277 +0000 UTC m=+0.380087912 container init e1a204f1394bbabb3adfa32b40f525da7fe7742532dc3e3e0381129245d4c3d7 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, RELEASE=main-56e81df, org.label-schema.vendor=CentOS) 2024-04-22T21:40:58.848 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 podman[75491]: 2024-04-22 21:40:58.481479822 +0000 UTC m=+0.383200469 container start e1a204f1394bbabb3adfa32b40f525da7fe7742532dc3e3e0381129245d4c3d7 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240416, RELEASE=main-56e81df, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-22T21:40:58.848 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 podman[75491]: 2024-04-22 21:40:58.561645178 +0000 UTC m=+0.463365825 container attach e1a204f1394bbabb3adfa32b40f525da7fe7742532dc3e3e0381129245d4c3d7 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, RELEASE=main-56e81df, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.schema-version=1.0) 2024-04-22T21:40:59.235 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:40:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:40:58] "GET /metrics HTTP/1.1" 200 38126 "" "Prometheus/2.43.0" 2024-04-22T21:40:59.236 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate[75505]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:40:59.236 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate[75505]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:40:59.236 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 bash[75491]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:40:59.236 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 bash[75491]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:40:59.236 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 bash[75491]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T21:40:59.236 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 bash[75491]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T21:40:59.236 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 bash[75491]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T21:40:59.236 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 bash[75491]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T21:40:59.236 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 bash[75491]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-22T21:40:59.237 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 bash[75491]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T21:40:59.237 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 bash[75491]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-22T21:40:59.237 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate[75505]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T21:40:59.237 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate[75505]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T21:40:59.237 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate[75505]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T21:40:59.237 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate[75505]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T21:40:59.237 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate[75505]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-22T21:40:59.237 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate[75505]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T21:40:59.237 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate[75505]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-22T21:40:59.237 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:58 smithi150 podman[75491]: 2024-04-22 21:40:58.986645169 +0000 UTC m=+0.888365810 container died e1a204f1394bbabb3adfa32b40f525da7fe7742532dc3e3e0381129245d4c3d7 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=main-56e81df, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-22T21:40:59.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:59 smithi130 ceph-mon[82829]: Health check failed: Degraded data redundancy: 98/627 objects degraded (15.630%), 24 pgs degraded (PG_DEGRADED) 2024-04-22T21:40:59.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:40:59 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:59.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:59 smithi130 ceph-mon[85205]: Health check failed: Degraded data redundancy: 98/627 objects degraded (15.630%), 24 pgs degraded (PG_DEGRADED) 2024-04-22T21:40:59.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:40:59 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:59.545 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:59 smithi150 ceph-mon[65103]: Health check failed: Degraded data redundancy: 98/627 objects degraded (15.630%), 24 pgs degraded (PG_DEGRADED) 2024-04-22T21:40:59.546 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:40:59 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:40:59.848 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:59 smithi150 podman[75644]: 2024-04-22 21:40:59.565547547 +0000 UTC m=+0.570494319 container remove e1a204f1394bbabb3adfa32b40f525da7fe7742532dc3e3e0381129245d4c3d7 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-56e81df, CEPH_POINT_RELEASE=) 2024-04-22T21:40:59.848 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:59 smithi150 podman[75705]: 2024-04-22 21:40:59.681919122 +0000 UTC m=+0.018836377 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:40:59.848 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:40:59 smithi150 podman[75705]: 2024-04-22 21:40:59.808095479 +0000 UTC m=+0.145012727 container create 0c80f1b448cbfa6c485d32c41895f89f4e7c83d81c4d98a6ceaa22dd3f81abf4 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-22T21:41:00.244 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:41:00 smithi150 podman[75705]: 2024-04-22 21:41:00.018668843 +0000 UTC m=+0.355586092 container init 0c80f1b448cbfa6c485d32c41895f89f4e7c83d81c4d98a6ceaa22dd3f81abf4 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:41:00.245 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:41:00 smithi150 podman[75705]: 2024-04-22 21:41:00.0215218 +0000 UTC m=+0.358439051 container start 0c80f1b448cbfa6c485d32c41895f89f4e7c83d81c4d98a6ceaa22dd3f81abf4 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4, org.label-schema.license=GPLv2, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-22T21:41:00.245 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:41:00 smithi150 ceph-osd[75725]: -- 172.21.15.150:0/1929425717 <== mon.2 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x565273881680 con 0x56527384b000 2024-04-22T21:41:00.245 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:41:00 smithi150 bash[75705]: 0c80f1b448cbfa6c485d32c41895f89f4e7c83d81c4d98a6ceaa22dd3f81abf4 2024-04-22T21:41:00.245 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:41:00 smithi150 systemd[1]: Started Ceph osd.4 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:41:00.542 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:00 smithi150 ceph-mon[65103]: pgmap v172: 161 pgs: 26 active+undersized+degraded, 3 stale+active+clean, 36 active+undersized, 96 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 103/627 objects degraded (16.427%) 2024-04-22T21:41:00.542 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:00 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:00.542 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:00 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:00.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:00 smithi130 ceph-mon[82829]: pgmap v172: 161 pgs: 26 active+undersized+degraded, 3 stale+active+clean, 36 active+undersized, 96 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 103/627 objects degraded (16.427%) 2024-04-22T21:41:00.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:00 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:00.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:00 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:00.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:00 smithi130 ceph-mon[85205]: pgmap v172: 161 pgs: 26 active+undersized+degraded, 3 stale+active+clean, 36 active+undersized, 96 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 103/627 objects degraded (16.427%) 2024-04-22T21:41:00.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:00 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:00.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:00 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:00.848 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:41:00 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:41:00.560+0000 7ff3de2557c0 -1 Falling back to public interface 2024-04-22T21:41:02.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:01 smithi130 ceph-mon[82829]: pgmap v173: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 109/627 objects degraded (17.384%) 2024-04-22T21:41:02.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:01 smithi130 ceph-mon[85205]: pgmap v173: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 109/627 objects degraded (17.384%) 2024-04-22T21:41:02.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:01 smithi150 ceph-mon[65103]: pgmap v173: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 109/627 objects degraded (17.384%) 2024-04-22T21:41:03.538 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:03 smithi150 ceph-mon[65103]: from='osd.4 [v2:172.21.15.150:6800/2810440127,v1:172.21.15.150:6801/2810440127]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T21:41:03.538 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:03 smithi150 ceph-mon[65103]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T21:41:03.539 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:41:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:41:03.279+0000 7ff3de2557c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-04-22T21:41:03.539 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:41:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:41:03.317+0000 7ff3de2557c0 -1 osd.4 133 log_to_monitors true 2024-04-22T21:41:03.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:03 smithi130 ceph-mon[82829]: from='osd.4 [v2:172.21.15.150:6800/2810440127,v1:172.21.15.150:6801/2810440127]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T21:41:03.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:03 smithi130 ceph-mon[82829]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T21:41:03.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:03 smithi130 ceph-mon[85205]: from='osd.4 [v2:172.21.15.150:6800/2810440127,v1:172.21.15.150:6801/2810440127]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T21:41:03.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:03 smithi130 ceph-mon[85205]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T21:41:04.173 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:41:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:41:03.872+0000 7ff3cb6a5700 -1 osd.4 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:41:04.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:04 smithi150 ceph-mon[65103]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T21:41:04.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:04 smithi150 ceph-mon[65103]: osdmap e136: 8 total, 7 up, 8 in 2024-04-22T21:41:04.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:04 smithi150 ceph-mon[65103]: from='osd.4 [v2:172.21.15.150:6800/2810440127,v1:172.21.15.150:6801/2810440127]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:41:04.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:04 smithi150 ceph-mon[65103]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:41:04.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:04 smithi150 ceph-mon[65103]: pgmap v175: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 109/627 objects degraded (17.384%) 2024-04-22T21:41:04.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:04 smithi150 ceph-mon[65103]: from='osd.4 ' entity='osd.4' 2024-04-22T21:41:04.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:04 smithi130 ceph-mon[82829]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T21:41:04.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:04 smithi130 ceph-mon[82829]: osdmap e136: 8 total, 7 up, 8 in 2024-04-22T21:41:04.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:04 smithi130 ceph-mon[82829]: from='osd.4 [v2:172.21.15.150:6800/2810440127,v1:172.21.15.150:6801/2810440127]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:41:04.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:04 smithi130 ceph-mon[82829]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:41:04.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:04 smithi130 ceph-mon[82829]: pgmap v175: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 109/627 objects degraded (17.384%) 2024-04-22T21:41:04.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:04 smithi130 ceph-mon[82829]: from='osd.4 ' entity='osd.4' 2024-04-22T21:41:04.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:04 smithi130 ceph-mon[85205]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T21:41:04.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:04 smithi130 ceph-mon[85205]: osdmap e136: 8 total, 7 up, 8 in 2024-04-22T21:41:04.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:04 smithi130 ceph-mon[85205]: from='osd.4 [v2:172.21.15.150:6800/2810440127,v1:172.21.15.150:6801/2810440127]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:41:04.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:04 smithi130 ceph-mon[85205]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:41:04.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:04 smithi130 ceph-mon[85205]: pgmap v175: 161 pgs: 28 active+undersized+degraded, 40 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 109/627 objects degraded (17.384%) 2024-04-22T21:41:04.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:04 smithi130 ceph-mon[85205]: from='osd.4 ' entity='osd.4' 2024-04-22T21:41:05.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:05 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:05.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:05 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:05.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:05 smithi150 ceph-mon[65103]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:41:05.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:05 smithi150 ceph-mon[65103]: osd.4 [v2:172.21.15.150:6800/2810440127,v1:172.21.15.150:6801/2810440127] boot 2024-04-22T21:41:05.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:05 smithi150 ceph-mon[65103]: osdmap e137: 8 total, 8 up, 8 in 2024-04-22T21:41:05.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:05 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:41:06.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:05 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:06.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:05 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:06.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:05 smithi130 ceph-mon[82829]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:41:06.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:05 smithi130 ceph-mon[82829]: osd.4 [v2:172.21.15.150:6800/2810440127,v1:172.21.15.150:6801/2810440127] boot 2024-04-22T21:41:06.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:05 smithi130 ceph-mon[82829]: osdmap e137: 8 total, 8 up, 8 in 2024-04-22T21:41:06.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:05 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:41:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:05 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:05 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:05 smithi130 ceph-mon[85205]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:41:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:05 smithi130 ceph-mon[85205]: osd.4 [v2:172.21.15.150:6800/2810440127,v1:172.21.15.150:6801/2810440127] boot 2024-04-22T21:41:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:05 smithi130 ceph-mon[85205]: osdmap e137: 8 total, 8 up, 8 in 2024-04-22T21:41:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:05 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T21:41:07.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:07.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:07.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:06 smithi130 ceph-mon[82829]: pgmap v177: 161 pgs: 3 peering, 28 active+undersized+degraded, 37 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 109/627 objects degraded (17.384%) 2024-04-22T21:41:07.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:41:07.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:07.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:06 smithi130 ceph-mon[82829]: osdmap e138: 8 total, 8 up, 8 in 2024-04-22T21:41:07.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:07.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:07.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:06 smithi130 ceph-mon[85205]: pgmap v177: 161 pgs: 3 peering, 28 active+undersized+degraded, 37 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 109/627 objects degraded (17.384%) 2024-04-22T21:41:07.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:41:07.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:07.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:06 smithi130 ceph-mon[85205]: osdmap e138: 8 total, 8 up, 8 in 2024-04-22T21:41:07.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:07.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:07.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:06 smithi150 ceph-mon[65103]: pgmap v177: 161 pgs: 3 peering, 28 active+undersized+degraded, 37 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 109/627 objects degraded (17.384%) 2024-04-22T21:41:07.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:41:07.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:07.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:06 smithi150 ceph-mon[65103]: osdmap e138: 8 total, 8 up, 8 in 2024-04-22T21:41:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:07 smithi130 ceph-mon[85205]: Health check update: Degraded data redundancy: 109/627 objects degraded (17.384%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T21:41:08.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:07 smithi130 ceph-mon[82829]: Health check update: Degraded data redundancy: 109/627 objects degraded (17.384%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T21:41:08.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:07 smithi150 ceph-mon[65103]: Health check update: Degraded data redundancy: 109/627 objects degraded (17.384%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T21:41:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:08 smithi130 ceph-mon[85205]: pgmap v179: 161 pgs: 3 peering, 11 active+undersized+degraded, 16 active+undersized, 131 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 32/627 objects degraded (5.104%) 2024-04-22T21:41:09.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:08 smithi130 ceph-mon[82829]: pgmap v179: 161 pgs: 3 peering, 11 active+undersized+degraded, 16 active+undersized, 131 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 32/627 objects degraded (5.104%) 2024-04-22T21:41:09.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:41:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:41:08] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.43.0" 2024-04-22T21:41:09.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:08 smithi150 ceph-mon[65103]: pgmap v179: 161 pgs: 3 peering, 11 active+undersized+degraded, 16 active+undersized, 131 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 32/627 objects degraded (5.104%) 2024-04-22T21:41:09.716 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:41:09.473+0000 7fd7212f0700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-22T21:41:10.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:10.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:10.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:10.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:41:10.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:41:10.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:10.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:41:10.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:10.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:10.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:10.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T21:41:10.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T21:41:10.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-22T21:41:10.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[82829]: pgmap v180: 161 pgs: 3 peering, 158 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T21:41:10.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:10.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:10.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:10.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:41:10.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:41:10.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:10.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:41:10.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:10.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:10.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:10.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T21:41:10.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T21:41:10.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-22T21:41:10.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:09 smithi130 ceph-mon[85205]: pgmap v180: 161 pgs: 3 peering, 158 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T21:41:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:41:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:41:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:41:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:10.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T21:41:10.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T21:41:10.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-22T21:41:10.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:09 smithi150 ceph-mon[65103]: pgmap v180: 161 pgs: 3 peering, 158 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T21:41:11.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:10 smithi130 ceph-mon[82829]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/627 objects degraded (5.104%), 11 pgs degraded) 2024-04-22T21:41:11.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:10 smithi130 ceph-mon[82829]: Cluster is now healthy 2024-04-22T21:41:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:10 smithi130 ceph-mon[85205]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/627 objects degraded (5.104%), 11 pgs degraded) 2024-04-22T21:41:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:10 smithi130 ceph-mon[85205]: Cluster is now healthy 2024-04-22T21:41:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:10 smithi150 ceph-mon[65103]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/627 objects degraded (5.104%), 11 pgs degraded) 2024-04-22T21:41:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:10 smithi150 ceph-mon[65103]: Cluster is now healthy 2024-04-22T21:41:12.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:11 smithi130 ceph-mon[85205]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:12.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:11 smithi130 ceph-mon[82829]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:12.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:11 smithi150 ceph-mon[65103]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:14.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:13 smithi130 ceph-mon[82829]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 578 B/s rd, 0 op/s 2024-04-22T21:41:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:13 smithi130 ceph-mon[85205]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 578 B/s rd, 0 op/s 2024-04-22T21:41:14.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:13 smithi150 ceph-mon[65103]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 578 B/s rd, 0 op/s 2024-04-22T21:41:16.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:15 smithi130 ceph-mon[82829]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:41:16.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:15 smithi130 ceph-mon[85205]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:41:16.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:15 smithi150 ceph-mon[65103]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:41:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:17 smithi130 ceph-mon[85205]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-04-22T21:41:18.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:17 smithi130 ceph-mon[82829]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-04-22T21:41:18.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:17 smithi150 ceph-mon[65103]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-04-22T21:41:19.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:18 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:19.045 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:41:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:41:18] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.43.0" 2024-04-22T21:41:19.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:18 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:19.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:18 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:20.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:19 smithi130 ceph-mon[82829]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:20.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:19 smithi130 ceph-mon[85205]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:20.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:19 smithi150 ceph-mon[65103]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:20.581 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:41:21.240 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (7m) 51s ago 29m 27.7M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (7m) 16s ago 29m 82.0M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (7m) 51s ago 29m 60.5M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (9m) 16s ago 34m 453M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (18m) 51s ago 36m 510M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (5m) 51s ago 36m 55.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (6m) 16s ago 34m 47.2M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (5m) 51s ago 35m 41.2M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (8m) 51s ago 30m 15.7M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (8m) 16s ago 30m 16.4M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (3m) 51s ago 34m 47.6M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (2m) 51s ago 34m 52.4M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (110s) 51s ago 33m 41.5M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 027e1ac8c177 2024-04-22T21:41:21.241 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (56s) 51s ago 33m 12.1M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 ca6c16e44590 2024-04-22T21:41:21.242 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (21s) 16s ago 32m 12.2M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 0c80f1b448cb 2024-04-22T21:41:21.242 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (32m) 16s ago 32m 56.0M 3248M 17.2.0 e1d6a67b021e 0de940bbe1a9 2024-04-22T21:41:21.242 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (32m) 16s ago 32m 58.7M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:41:21.242 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (31m) 16s ago 31m 59.0M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:41:21.242 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (8m) 16s ago 30m 71.9M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:41:21.242 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (29m) 51s ago 29m 80.2M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:41:21.242 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (29m) 16s ago 29m 77.7M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:41:21.641 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:41:21.641 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:41:21.642 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:41:21.642 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:41:21.642 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:41:21.642 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:41:21.642 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:41:21.642 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:41:21.643 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-22T21:41:21.643 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 5 2024-04-22T21:41:21.643 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:41:21.643 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:41:21.643 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:41:21.643 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:41:21.644 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:41:21.644 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-22T21:41:21.644 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 10 2024-04-22T21:41:21.644 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:41:21.644 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:41:21.979 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:41:21.979 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:41:21.979 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:41:21.979 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-22T21:41:21.980 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:41:21.980 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "5/8 daemons upgraded", 2024-04-22T21:41:21.980 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T21:41:21.980 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:41:21.980 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:41:21.980 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:21 smithi130 ceph-mon[85205]: from='client.64441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:21.980 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:21.980 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:41:21.980 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:21.981 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:21 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/1529123316' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:21.981 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:21 smithi130 ceph-mon[82829]: from='client.64441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:21.981 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:21.981 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:41:21.981 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:21.981 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:21 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/1529123316' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:22.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:21 smithi150 ceph-mon[65103]: from='client.64441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:22.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:22.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:41:22.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:22.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:21 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/1529123316' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:23.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:22 smithi130 ceph-mon[85205]: from='client.64447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:23.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:22 smithi130 ceph-mon[85205]: from='client.64453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:23.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:22 smithi130 ceph-mon[85205]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:41:23.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:22 smithi130 ceph-mon[82829]: from='client.64447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:23.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:22 smithi130 ceph-mon[82829]: from='client.64453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:23.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:22 smithi130 ceph-mon[82829]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:41:23.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:22 smithi150 ceph-mon[65103]: from='client.64447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:23.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:22 smithi150 ceph-mon[65103]: from='client.64453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:23.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:22 smithi150 ceph-mon[65103]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:41:23.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:23 smithi130 ceph-mon[82829]: from='client.54531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:23.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:23 smithi130 ceph-mon[85205]: from='client.54531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:24.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:23 smithi150 ceph-mon[65103]: from='client.54531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:25.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:24 smithi130 ceph-mon[82829]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:41:25.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:24 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T21:41:25.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:24 smithi130 ceph-mon[85205]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:41:25.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:24 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T21:41:25.058 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:24 smithi150 ceph-mon[65103]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:41:25.058 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:24 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T21:41:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:25 smithi130 ceph-mon[85205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T21:41:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:25 smithi130 ceph-mon[85205]: Upgrade: osd.5 is safe to restart 2024-04-22T21:41:26.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:25 smithi130 ceph-mon[82829]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T21:41:26.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:25 smithi130 ceph-mon[82829]: Upgrade: osd.5 is safe to restart 2024-04-22T21:41:26.079 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:25 smithi150 ceph-mon[65103]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T21:41:26.079 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:25 smithi150 ceph-mon[65103]: Upgrade: osd.5 is safe to restart 2024-04-22T21:41:26.978 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:26 smithi150 ceph-mon[65103]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:26.978 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:26 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:26.978 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:26 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T21:41:26.978 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:26 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:41:27.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:26 smithi130 ceph-mon[82829]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:27.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:26 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:27.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:26 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T21:41:27.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:26 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:41:27.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:26 smithi130 ceph-mon[85205]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:27.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:26 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:27.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:26 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T21:41:27.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:26 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:41:28.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:27 smithi130 ceph-mon[82829]: Upgrade: Updating osd.5 2024-04-22T21:41:28.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:27 smithi130 ceph-mon[82829]: Deploying daemon osd.5 on smithi150 2024-04-22T21:41:28.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:27 smithi130 ceph-mon[85205]: Upgrade: Updating osd.5 2024-04-22T21:41:28.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:27 smithi130 ceph-mon[85205]: Deploying daemon osd.5 on smithi150 2024-04-22T21:41:28.059 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:27 smithi150 ceph-mon[65103]: Upgrade: Updating osd.5 2024-04-22T21:41:28.059 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:27 smithi150 ceph-mon[65103]: Deploying daemon osd.5 on smithi150 2024-04-22T21:41:29.005 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:28 smithi150 ceph-mon[65103]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:41:29.006 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:28 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:29.044 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:41:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:41:28] "GET /metrics HTTP/1.1" 200 38131 "" "Prometheus/2.43.0" 2024-04-22T21:41:29.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:28 smithi130 ceph-mon[82829]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:41:29.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:28 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:29.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:28 smithi130 ceph-mon[85205]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:41:29.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:28 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:29.598 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:29 smithi150 systemd[1]: Stopping Ceph osd.5 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:41:30.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:29 smithi130 ceph-mon[82829]: osd.5 marked itself down and dead 2024-04-22T21:41:30.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:29 smithi130 ceph-mon[82829]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:30.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:29 smithi130 ceph-mon[85205]: osd.5 marked itself down and dead 2024-04-22T21:41:30.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:29 smithi130 ceph-mon[85205]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:30.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:29 smithi150 ceph-mon[65103]: osd.5 marked itself down and dead 2024-04-22T21:41:30.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:29 smithi150 ceph-mon[65103]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:30.099 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[28986]: 2024-04-22T21:41:29.637+0000 7f257a7ad700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:41:30.099 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[28986]: 2024-04-22T21:41:29.637+0000 7f257a7ad700 -1 osd.5 138 *** Got signal Terminated *** 2024-04-22T21:41:30.099 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[28986]: 2024-04-22T21:41:29.637+0000 7f257a7ad700 -1 osd.5 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:41:30.395 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:30 smithi150 podman[79804]: 2024-04-22 21:41:30.128699805 +0000 UTC m=+0.618186611 container died 0de940bbe1a9af7508b168646e0e5d5d9477d14863ec8519a705111273fe86b6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, ceph=True, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, distribution-scope=public, maintainer=Guillaume Abrioux , release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, architecture=x86_64) 2024-04-22T21:41:30.723 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:30 smithi150 podman[79804]: 2024-04-22 21:41:30.521435426 +0000 UTC m=+1.010922228 container cleanup 0de940bbe1a9af7508b168646e0e5d5d9477d14863ec8519a705111273fe86b6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, name=centos-stream, io.openshift.expose-services=, release=754, com.redhat.component=centos-stream-container, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git) 2024-04-22T21:41:30.723 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:30 smithi150 bash[79804]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5 2024-04-22T21:41:31.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:30 smithi130 ceph-mon[82829]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:41:31.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:30 smithi130 ceph-mon[82829]: osdmap e139: 8 total, 7 up, 8 in 2024-04-22T21:41:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:30 smithi130 ceph-mon[85205]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:41:31.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:30 smithi130 ceph-mon[85205]: osdmap e139: 8 total, 7 up, 8 in 2024-04-22T21:41:31.084 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:30 smithi150 ceph-mon[65103]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:41:31.084 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:30 smithi150 ceph-mon[65103]: osdmap e139: 8 total, 7 up, 8 in 2024-04-22T21:41:31.085 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:30 smithi150 podman[79822]: 2024-04-22 21:41:30.789632422 +0000 UTC m=+0.661340466 container remove 0de940bbe1a9af7508b168646e0e5d5d9477d14863ec8519a705111273fe86b6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.tags=base centos centos-stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, release=754, version=8, architecture=x86_64, io.buildah.version=1.19.8) 2024-04-22T21:41:31.348 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:31 smithi150 podman[79915]: 2024-04-22 21:41:31.006840945 +0000 UTC m=+0.018624659 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:41:31.348 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:31 smithi150 podman[79915]: 2024-04-22 21:41:31.123868931 +0000 UTC m=+0.135652676 container create 17184e593bfa32aeed99cf374a390ca7eea1dedb9efe719e991491136024c2da (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-56e81df, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:41:31.651 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:31 smithi150 podman[79915]: 2024-04-22 21:41:31.368656981 +0000 UTC m=+0.380440690 container init 17184e593bfa32aeed99cf374a390ca7eea1dedb9efe719e991491136024c2da (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, RELEASE=main-56e81df, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-22T21:41:31.651 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:31 smithi150 podman[79915]: 2024-04-22 21:41:31.373293099 +0000 UTC m=+0.385076803 container start 17184e593bfa32aeed99cf374a390ca7eea1dedb9efe719e991491136024c2da (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-56e81df, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:41:31.651 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:31 smithi150 podman[79915]: 2024-04-22 21:41:31.459649346 +0000 UTC m=+0.471433054 container attach 17184e593bfa32aeed99cf374a390ca7eea1dedb9efe719e991491136024c2da (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-22T21:41:31.963 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:31 smithi150 ceph-mon[65103]: osdmap e140: 8 total, 7 up, 8 in 2024-04-22T21:41:31.963 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:31 smithi150 ceph-mon[65103]: pgmap v193: 161 pgs: 8 active+undersized+degraded, 7 peering, 10 stale+active+clean, 19 active+undersized, 117 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-04-22T21:41:31.963 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:31 smithi150 podman[79915]: 2024-04-22 21:41:31.670464958 +0000 UTC m=+0.682248663 container died 17184e593bfa32aeed99cf374a390ca7eea1dedb9efe719e991491136024c2da (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:41:32.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:31 smithi130 ceph-mon[82829]: osdmap e140: 8 total, 7 up, 8 in 2024-04-22T21:41:32.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:31 smithi130 ceph-mon[82829]: pgmap v193: 161 pgs: 8 active+undersized+degraded, 7 peering, 10 stale+active+clean, 19 active+undersized, 117 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-04-22T21:41:32.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:31 smithi130 ceph-mon[85205]: osdmap e140: 8 total, 7 up, 8 in 2024-04-22T21:41:32.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:31 smithi130 ceph-mon[85205]: pgmap v193: 161 pgs: 8 active+undersized+degraded, 7 peering, 10 stale+active+clean, 19 active+undersized, 117 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-04-22T21:41:32.598 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:32 smithi150 podman[79941]: 2024-04-22 21:41:32.248771682 +0000 UTC m=+0.573156361 container remove 17184e593bfa32aeed99cf374a390ca7eea1dedb9efe719e991491136024c2da (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-56e81df) 2024-04-22T21:41:32.599 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:32 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.5.service: Deactivated successfully. 2024-04-22T21:41:32.599 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:32 smithi150 systemd[1]: Stopped Ceph osd.5 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:41:32.599 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:32 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.5.service: Consumed 16.837s CPU time. 2024-04-22T21:41:32.599 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:32 smithi150 systemd[1]: Starting Ceph osd.5 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:41:32.976 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:32 smithi150 podman[80046]: 2024-04-22 21:41:32.591440492 +0000 UTC m=+0.018388872 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:41:32.976 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:32 smithi150 podman[80046]: 2024-04-22 21:41:32.733562017 +0000 UTC m=+0.160510397 container create e42ac492ba3da150ea2ebe7d97aa84b6466f96c0e3b3212f69c9455ccd665db3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-56e81df, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, ceph=True) 2024-04-22T21:41:32.976 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:32 smithi150 ceph-mon[65103]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-22T21:41:32.976 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:32 smithi150 ceph-mon[65103]: Health check failed: Degraded data redundancy: 28/627 objects degraded (4.466%), 8 pgs degraded (PG_DEGRADED) 2024-04-22T21:41:33.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:32 smithi130 ceph-mon[82829]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-22T21:41:33.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:32 smithi130 ceph-mon[82829]: Health check failed: Degraded data redundancy: 28/627 objects degraded (4.466%), 8 pgs degraded (PG_DEGRADED) 2024-04-22T21:41:33.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:32 smithi130 ceph-mon[85205]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-22T21:41:33.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:32 smithi130 ceph-mon[85205]: Health check failed: Degraded data redundancy: 28/627 objects degraded (4.466%), 8 pgs degraded (PG_DEGRADED) 2024-04-22T21:41:33.348 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:32 smithi150 podman[80046]: 2024-04-22 21:41:32.995000987 +0000 UTC m=+0.421949367 container init e42ac492ba3da150ea2ebe7d97aa84b6466f96c0e3b3212f69c9455ccd665db3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True) 2024-04-22T21:41:33.348 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:32 smithi150 podman[80046]: 2024-04-22 21:41:32.999285011 +0000 UTC m=+0.426233393 container start e42ac492ba3da150ea2ebe7d97aa84b6466f96c0e3b3212f69c9455ccd665db3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-56e81df, ceph=True) 2024-04-22T21:41:33.348 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 podman[80046]: 2024-04-22 21:41:33.103468471 +0000 UTC m=+0.530416861 container attach e42ac492ba3da150ea2ebe7d97aa84b6466f96c0e3b3212f69c9455ccd665db3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, org.label-schema.license=GPLv2) 2024-04-22T21:41:33.744 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate[80061]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:41:33.744 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate[80061]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:41:33.744 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 bash[80046]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:41:33.744 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 bash[80046]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:41:33.744 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 bash[80046]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T21:41:33.744 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 bash[80046]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T21:41:33.744 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 bash[80046]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T21:41:33.744 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 bash[80046]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T21:41:33.744 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 bash[80046]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-22T21:41:33.744 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 bash[80046]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T21:41:33.745 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 bash[80046]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-22T21:41:33.745 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate[80061]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T21:41:33.745 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate[80061]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T21:41:33.745 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate[80061]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T21:41:33.745 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate[80061]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T21:41:33.745 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate[80061]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-22T21:41:33.745 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate[80061]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T21:41:33.745 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate[80061]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-22T21:41:33.745 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:33 smithi150 podman[80046]: 2024-04-22 21:41:33.497527867 +0000 UTC m=+0.924476248 container died e42ac492ba3da150ea2ebe7d97aa84b6466f96c0e3b3212f69c9455ccd665db3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:41:34.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:33 smithi130 ceph-mon[82829]: pgmap v194: 161 pgs: 14 active+undersized+degraded, 7 peering, 5 stale+active+clean, 29 active+undersized, 106 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 51/627 objects degraded (8.134%) 2024-04-22T21:41:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:33 smithi130 ceph-mon[85205]: pgmap v194: 161 pgs: 14 active+undersized+degraded, 7 peering, 5 stale+active+clean, 29 active+undersized, 106 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 51/627 objects degraded (8.134%) 2024-04-22T21:41:34.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:33 smithi150 ceph-mon[65103]: pgmap v194: 161 pgs: 14 active+undersized+degraded, 7 peering, 5 stale+active+clean, 29 active+undersized, 106 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 51/627 objects degraded (8.134%) 2024-04-22T21:41:34.482 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:34 smithi150 podman[80183]: 2024-04-22 21:41:34.132710905 +0000 UTC m=+0.626283368 container remove e42ac492ba3da150ea2ebe7d97aa84b6466f96c0e3b3212f69c9455ccd665db3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , RELEASE=main-56e81df, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-22T21:41:34.482 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:34 smithi150 podman[80249]: 2024-04-22 21:41:34.252539348 +0000 UTC m=+0.018196853 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:41:34.482 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:34 smithi150 podman[80249]: 2024-04-22 21:41:34.358648224 +0000 UTC m=+0.124305743 container create aef1942f3f6becfefcd9237db20907ca1e3e80b208a9828af0ddc01f41d526eb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5, RELEASE=main-56e81df, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-22T21:41:35.098 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:34 smithi150 podman[80249]: 2024-04-22 21:41:34.636341422 +0000 UTC m=+0.401998902 container init aef1942f3f6becfefcd9237db20907ca1e3e80b208a9828af0ddc01f41d526eb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-56e81df, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240416, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-22T21:41:35.098 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:34 smithi150 podman[80249]: 2024-04-22 21:41:34.639802574 +0000 UTC m=+0.405460052 container start aef1942f3f6becfefcd9237db20907ca1e3e80b208a9828af0ddc01f41d526eb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-56e81df, org.label-schema.schema-version=1.0) 2024-04-22T21:41:35.098 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:34 smithi150 ceph-osd[80269]: -- 172.21.15.150:0/3491583977 <== mon.2 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563cc85e9680 con 0x563cc85b3400 2024-04-22T21:41:35.098 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:34 smithi150 bash[80249]: aef1942f3f6becfefcd9237db20907ca1e3e80b208a9828af0ddc01f41d526eb 2024-04-22T21:41:35.099 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:34 smithi150 systemd[1]: Started Ceph osd.5 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:41:35.596 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:41:35.181+0000 7ff972f5e7c0 -1 Falling back to public interface 2024-04-22T21:41:35.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:35.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:35.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:35.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:41:36.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:36.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:36.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:36.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:41:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:36.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:41:37.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:36 smithi130 ceph-mon[85205]: pgmap v195: 161 pgs: 19 active+undersized+degraded, 41 active+undersized, 101 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-04-22T21:41:37.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:36 smithi130 ceph-mon[85205]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-22T21:41:37.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:36 smithi130 ceph-mon[82829]: pgmap v195: 161 pgs: 19 active+undersized+degraded, 41 active+undersized, 101 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-04-22T21:41:37.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:36 smithi130 ceph-mon[82829]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-22T21:41:37.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:36 smithi150 ceph-mon[65103]: pgmap v195: 161 pgs: 19 active+undersized+degraded, 41 active+undersized, 101 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-04-22T21:41:37.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:36 smithi150 ceph-mon[65103]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-22T21:41:38.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:37 smithi130 ceph-mon[85205]: Health check update: Degraded data redundancy: 70/627 objects degraded (11.164%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T21:41:38.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:37 smithi130 ceph-mon[82829]: Health check update: Degraded data redundancy: 70/627 objects degraded (11.164%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T21:41:38.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:37 smithi150 ceph-mon[65103]: Health check update: Degraded data redundancy: 70/627 objects degraded (11.164%), 19 pgs degraded (PG_DEGRADED) 2024-04-22T21:41:38.098 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:37 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:41:37.886+0000 7ff972f5e7c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-04-22T21:41:38.099 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:37 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:41:37.917+0000 7ff972f5e7c0 -1 osd.5 138 log_to_monitors true 2024-04-22T21:41:39.008 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:38 smithi150 ceph-mon[65103]: pgmap v196: 161 pgs: 19 active+undersized+degraded, 41 active+undersized, 101 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-04-22T21:41:39.008 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:38 smithi150 ceph-mon[65103]: from='osd.5 [v2:172.21.15.150:6808/3913607130,v1:172.21.15.150:6809/3913607130]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T21:41:39.008 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:38 smithi150 ceph-mon[65103]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T21:41:39.009 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:38 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:39.009 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:41:38 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:41:38.827+0000 7ff9643b6700 -1 osd.5 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:41:39.044 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:41:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:41:38] "GET /metrics HTTP/1.1" 200 38199 "" "Prometheus/2.43.0" 2024-04-22T21:41:39.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:38 smithi130 ceph-mon[82829]: pgmap v196: 161 pgs: 19 active+undersized+degraded, 41 active+undersized, 101 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-04-22T21:41:39.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:38 smithi130 ceph-mon[82829]: from='osd.5 [v2:172.21.15.150:6808/3913607130,v1:172.21.15.150:6809/3913607130]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T21:41:39.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:38 smithi130 ceph-mon[82829]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T21:41:39.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:38 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:38 smithi130 ceph-mon[85205]: pgmap v196: 161 pgs: 19 active+undersized+degraded, 41 active+undersized, 101 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-04-22T21:41:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:38 smithi130 ceph-mon[85205]: from='osd.5 [v2:172.21.15.150:6808/3913607130,v1:172.21.15.150:6809/3913607130]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T21:41:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:38 smithi130 ceph-mon[85205]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T21:41:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:38 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:40.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:39 smithi150 ceph-mon[65103]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T21:41:40.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:39 smithi150 ceph-mon[65103]: from='osd.5 [v2:172.21.15.150:6808/3913607130,v1:172.21.15.150:6809/3913607130]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:41:40.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:39 smithi150 ceph-mon[65103]: osdmap e141: 8 total, 7 up, 8 in 2024-04-22T21:41:40.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:39 smithi150 ceph-mon[65103]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:41:40.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:39 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:40.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:39 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:40.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:39 smithi130 ceph-mon[82829]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T21:41:40.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:39 smithi130 ceph-mon[82829]: from='osd.5 [v2:172.21.15.150:6808/3913607130,v1:172.21.15.150:6809/3913607130]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:41:40.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:39 smithi130 ceph-mon[82829]: osdmap e141: 8 total, 7 up, 8 in 2024-04-22T21:41:40.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:39 smithi130 ceph-mon[82829]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:41:40.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:39 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:40.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:39 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:40.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:39 smithi130 ceph-mon[85205]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T21:41:40.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:39 smithi130 ceph-mon[85205]: from='osd.5 [v2:172.21.15.150:6808/3913607130,v1:172.21.15.150:6809/3913607130]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:41:40.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:39 smithi130 ceph-mon[85205]: osdmap e141: 8 total, 7 up, 8 in 2024-04-22T21:41:40.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:39 smithi130 ceph-mon[85205]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:41:40.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:39 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:40.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:39 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:41.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:40 smithi150 ceph-mon[65103]: pgmap v198: 161 pgs: 19 active+undersized+degraded, 41 active+undersized, 101 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-04-22T21:41:41.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:40 smithi150 ceph-mon[65103]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:41:41.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:40 smithi150 ceph-mon[65103]: osd.5 [v2:172.21.15.150:6808/3913607130,v1:172.21.15.150:6809/3913607130] boot 2024-04-22T21:41:41.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:40 smithi150 ceph-mon[65103]: osdmap e142: 8 total, 8 up, 8 in 2024-04-22T21:41:41.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:40 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:41:41.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:40 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:41.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:40 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:41.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:40 smithi150 ceph-mon[65103]: osdmap e143: 8 total, 8 up, 8 in 2024-04-22T21:41:41.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[82829]: pgmap v198: 161 pgs: 19 active+undersized+degraded, 41 active+undersized, 101 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-04-22T21:41:41.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[82829]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:41:41.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[82829]: osd.5 [v2:172.21.15.150:6808/3913607130,v1:172.21.15.150:6809/3913607130] boot 2024-04-22T21:41:41.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[82829]: osdmap e142: 8 total, 8 up, 8 in 2024-04-22T21:41:41.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:41:41.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:41.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:41.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[82829]: osdmap e143: 8 total, 8 up, 8 in 2024-04-22T21:41:41.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[85205]: pgmap v198: 161 pgs: 19 active+undersized+degraded, 41 active+undersized, 101 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 70/627 objects degraded (11.164%) 2024-04-22T21:41:41.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[85205]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:41:41.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[85205]: osd.5 [v2:172.21.15.150:6808/3913607130,v1:172.21.15.150:6809/3913607130] boot 2024-04-22T21:41:41.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[85205]: osdmap e142: 8 total, 8 up, 8 in 2024-04-22T21:41:41.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T21:41:41.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:41.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:41.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:40 smithi130 ceph-mon[85205]: osdmap e143: 8 total, 8 up, 8 in 2024-04-22T21:41:43.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:42 smithi150 ceph-mon[65103]: pgmap v201: 161 pgs: 5 peering, 14 active+undersized+degraded, 24 active+undersized, 118 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 48/627 objects degraded (7.656%) 2024-04-22T21:41:43.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:42 smithi150 ceph-mon[65103]: Health check update: Degraded data redundancy: 48/627 objects degraded (7.656%), 14 pgs degraded (PG_DEGRADED) 2024-04-22T21:41:43.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:42 smithi130 ceph-mon[82829]: pgmap v201: 161 pgs: 5 peering, 14 active+undersized+degraded, 24 active+undersized, 118 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 48/627 objects degraded (7.656%) 2024-04-22T21:41:43.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:42 smithi130 ceph-mon[82829]: Health check update: Degraded data redundancy: 48/627 objects degraded (7.656%), 14 pgs degraded (PG_DEGRADED) 2024-04-22T21:41:43.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:42 smithi130 ceph-mon[85205]: pgmap v201: 161 pgs: 5 peering, 14 active+undersized+degraded, 24 active+undersized, 118 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 48/627 objects degraded (7.656%) 2024-04-22T21:41:43.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:42 smithi130 ceph-mon[85205]: Health check update: Degraded data redundancy: 48/627 objects degraded (7.656%), 14 pgs degraded (PG_DEGRADED) 2024-04-22T21:41:44.544 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:41:44 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:41:44.216+0000 7fd7212f0700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-22T21:41:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[82829]: pgmap v202: 161 pgs: 5 peering, 10 active+undersized+degraded, 17 active+undersized, 129 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 38/627 objects degraded (6.061%) 2024-04-22T21:41:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:41:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:41:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:41:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T21:41:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[82829]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T21:41:45.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[82829]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-22T21:41:45.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[85205]: pgmap v202: 161 pgs: 5 peering, 10 active+undersized+degraded, 17 active+undersized, 129 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 38/627 objects degraded (6.061%) 2024-04-22T21:41:45.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:45.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:45.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:41:45.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:41:45.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:45.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:41:45.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:45.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:45.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:45.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T21:41:45.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[85205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T21:41:45.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:45 smithi130 ceph-mon[85205]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-22T21:41:45.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:45 smithi150 ceph-mon[65103]: pgmap v202: 161 pgs: 5 peering, 10 active+undersized+degraded, 17 active+undersized, 129 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 38/627 objects degraded (6.061%) 2024-04-22T21:41:45.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:45 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:45.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:45 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:45.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:45 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:41:45.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:45 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:41:45.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:45 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:45.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:45 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:41:45.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:45 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:45.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:45 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:45.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:45 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:45.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:45 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T21:41:45.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:45 smithi150 ceph-mon[65103]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T21:41:45.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:45 smithi150 ceph-mon[65103]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-22T21:41:46.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:46 smithi150 ceph-mon[65103]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/627 objects degraded (6.061%), 10 pgs degraded) 2024-04-22T21:41:46.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:46 smithi150 ceph-mon[65103]: Cluster is now healthy 2024-04-22T21:41:46.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:46 smithi130 ceph-mon[82829]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/627 objects degraded (6.061%), 10 pgs degraded) 2024-04-22T21:41:46.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:46 smithi130 ceph-mon[82829]: Cluster is now healthy 2024-04-22T21:41:46.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:46 smithi130 ceph-mon[85205]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/627 objects degraded (6.061%), 10 pgs degraded) 2024-04-22T21:41:46.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:46 smithi130 ceph-mon[85205]: Cluster is now healthy 2024-04-22T21:41:47.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:47 smithi150 ceph-mon[65103]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:47.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:47 smithi130 ceph-mon[82829]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:47.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:47 smithi130 ceph-mon[85205]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:41:49.294 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:41:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:41:48] "GET /metrics HTTP/1.1" 200 38199 "" "Prometheus/2.43.0" 2024-04-22T21:41:49.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:49 smithi130 ceph-mon[82829]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-04-22T21:41:49.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:49 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:49.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:49 smithi130 ceph-mon[85205]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-04-22T21:41:49.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:49 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:49.395 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:49 smithi150 ceph-mon[65103]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-04-22T21:41:49.396 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:49 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:52.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:51 smithi130 ceph-mon[82829]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 824 B/s rd, 0 op/s 2024-04-22T21:41:52.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:52.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:41:52.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:52.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:51 smithi130 ceph-mon[85205]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 824 B/s rd, 0 op/s 2024-04-22T21:41:52.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:52.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:41:52.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:52.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:51 smithi150 ceph-mon[65103]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 824 B/s rd, 0 op/s 2024-04-22T21:41:52.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:52.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:41:52.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:41:52.341 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:41:52.990 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:41:52.990 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (8m) 82s ago 30m 27.7M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:41:52.990 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (7m) 13s ago 30m 81.1M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:41:52.990 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (7m) 82s ago 29m 60.5M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:41:52.990 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (9m) 13s ago 35m 453M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:41:52.990 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (19m) 82s ago 36m 510M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:41:52.990 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (5m) 82s ago 36m 55.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:41:52.990 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (6m) 13s ago 35m 48.6M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:41:52.990 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (5m) 82s ago 35m 41.2M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:41:52.990 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (9m) 82s ago 31m 15.7M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:41:52.990 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (9m) 13s ago 31m 15.2M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:41:52.991 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (3m) 82s ago 34m 47.6M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:41:52.991 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (3m) 82s ago 34m 52.4M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:41:52.991 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (2m) 82s ago 34m 41.5M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 027e1ac8c177 2024-04-22T21:41:52.991 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (87s) 82s ago 33m 12.1M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 ca6c16e44590 2024-04-22T21:41:52.991 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (53s) 13s ago 33m 50.9M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 0c80f1b448cb 2024-04-22T21:41:52.991 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (18s) 13s ago 32m 14.9M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 aef1942f3f6b 2024-04-22T21:41:52.991 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (32m) 13s ago 32m 59.5M 3248M 17.2.0 e1d6a67b021e 902636c1a0e4 2024-04-22T21:41:52.991 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (32m) 13s ago 32m 59.7M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:41:52.991 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (8m) 13s ago 31m 71.9M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:41:52.991 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (29m) 82s ago 29m 80.2M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:41:52.991 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (29m) 13s ago 29m 77.9M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:41:53.385 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:41:53.385 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:41:53.385 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:41:53.385 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:41:53.385 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 6 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 11 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:41:53.386 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:41:53.722 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:41:53.723 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:41:53.723 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:41:53.723 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-22T21:41:53.723 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:41:53.723 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "6/8 daemons upgraded", 2024-04-22T21:41:53.723 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T21:41:53.723 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:41:53.723 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:41:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:53 smithi130 ceph-mon[85205]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 748 B/s rd, 0 op/s 2024-04-22T21:41:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:53 smithi130 ceph-mon[85205]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:53 smithi130 ceph-mon[85205]: from='client.64477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:53 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/3428149955' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:54.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:53 smithi130 ceph-mon[82829]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 748 B/s rd, 0 op/s 2024-04-22T21:41:54.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:53 smithi130 ceph-mon[82829]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:54.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:53 smithi130 ceph-mon[82829]: from='client.64477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:54.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:53 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/3428149955' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:54.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:53 smithi150 ceph-mon[65103]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 748 B/s rd, 0 op/s 2024-04-22T21:41:54.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:53 smithi150 ceph-mon[65103]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:54.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:53 smithi150 ceph-mon[65103]: from='client.64477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:54.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:53 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/3428149955' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:41:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:54 smithi130 ceph-mon[85205]: from='client.54549 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:55.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:54 smithi130 ceph-mon[82829]: from='client.54549 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:55.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:54 smithi150 ceph-mon[65103]: from='client.54549 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:55 smithi130 ceph-mon[85205]: from='client.54561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:55 smithi130 ceph-mon[85205]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-22T21:41:56.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:55 smithi130 ceph-mon[82829]: from='client.54561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:56.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:55 smithi130 ceph-mon[82829]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-22T21:41:56.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:55 smithi150 ceph-mon[65103]: from='client.54561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:41:56.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:55 smithi150 ceph-mon[65103]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-22T21:41:58.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:57 smithi130 ceph-mon[82829]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T21:41:58.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:57 smithi130 ceph-mon[85205]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T21:41:58.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:57 smithi150 ceph-mon[65103]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T21:41:59.294 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:41:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:41:58] "GET /metrics HTTP/1.1" 200 38212 "" "Prometheus/2.43.0" 2024-04-22T21:41:59.998 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:59 smithi150 ceph-mon[65103]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:41:59.998 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:59 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:41:59.998 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:41:59 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T21:42:00.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:59 smithi130 ceph-mon[82829]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:00.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:59 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:00.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:41:59 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T21:42:00.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:59 smithi130 ceph-mon[85205]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:00.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:59 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:00.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:41:59 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T21:42:01.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:00 smithi130 ceph-mon[82829]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T21:42:01.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:00 smithi130 ceph-mon[82829]: Upgrade: osd.6 is safe to restart 2024-04-22T21:42:01.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:00 smithi130 ceph-mon[85205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T21:42:01.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:00 smithi130 ceph-mon[85205]: Upgrade: osd.6 is safe to restart 2024-04-22T21:42:01.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:00 smithi150 ceph-mon[65103]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T21:42:01.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:00 smithi150 ceph-mon[65103]: Upgrade: osd.6 is safe to restart 2024-04-22T21:42:01.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:01 smithi150 ceph-mon[65103]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:42:01.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:01 smithi150 ceph-mon[65103]: Upgrade: Updating osd.6 2024-04-22T21:42:01.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:01 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:01.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:01 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T21:42:01.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:01 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:01.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:01 smithi150 ceph-mon[65103]: Deploying daemon osd.6 on smithi150 2024-04-22T21:42:02.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:01 smithi130 ceph-mon[82829]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:42:02.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:01 smithi130 ceph-mon[82829]: Upgrade: Updating osd.6 2024-04-22T21:42:02.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:01 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:02.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:01 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T21:42:02.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:01 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:02.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:01 smithi130 ceph-mon[82829]: Deploying daemon osd.6 on smithi150 2024-04-22T21:42:02.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:01 smithi130 ceph-mon[85205]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:42:02.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:01 smithi130 ceph-mon[85205]: Upgrade: Updating osd.6 2024-04-22T21:42:02.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:01 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:02.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:01 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T21:42:02.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:01 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:02.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:01 smithi130 ceph-mon[85205]: Deploying daemon osd.6 on smithi150 2024-04-22T21:42:03.237 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:03 smithi150 ceph-mon[65103]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:03.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:03 smithi130 ceph-mon[82829]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:03.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:03 smithi130 ceph-mon[85205]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:04.485 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:04 smithi150 systemd[1]: Stopping Ceph osd.6 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:42:04.848 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[31883]: 2024-04-22T21:42:04.503+0000 7f040ee03700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:42:04.848 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[31883]: 2024-04-22T21:42:04.503+0000 7f040ee03700 -1 osd.6 143 *** Got signal Terminated *** 2024-04-22T21:42:04.848 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[31883]: 2024-04-22T21:42:04.503+0000 7f040ee03700 -1 osd.6 143 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:42:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:05 smithi130 ceph-mon[85205]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:05.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:05 smithi130 ceph-mon[85205]: osd.6 marked itself down and dead 2024-04-22T21:42:05.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:05 smithi130 ceph-mon[82829]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:05.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:05 smithi130 ceph-mon[82829]: osd.6 marked itself down and dead 2024-04-22T21:42:05.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:05 smithi150 ceph-mon[65103]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:05.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:05 smithi150 ceph-mon[65103]: osd.6 marked itself down and dead 2024-04-22T21:42:05.719 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:05 smithi150 podman[84795]: 2024-04-22 21:42:05.475985381 +0000 UTC m=+1.086614627 container died 902636c1a0e44e14dbcee737a30c35922f852d0bc21b328ab7f133bbd4016252 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6, release=754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, RELEASE=HEAD, ceph=True, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, vcs-type=git, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, vendor=Red Hat, Inc., distribution-scope=public, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, io.k8s.display-name=CentOS Stream 8) 2024-04-22T21:42:05.992 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:05 smithi150 podman[84795]: 2024-04-22 21:42:05.845934663 +0000 UTC m=+1.456563916 container cleanup 902636c1a0e44e14dbcee737a30c35922f852d0bc21b328ab7f133bbd4016252 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6, release=754, io.k8s.display-name=CentOS Stream 8, version=8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, RELEASE=HEAD, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, name=centos-stream, ceph=True, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-22T21:42:05.992 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:05 smithi150 bash[84795]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6 2024-04-22T21:42:06.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:42:06.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: osdmap e144: 8 total, 7 up, 8 in 2024-04-22T21:42:06.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.9", "id": [7, 2]}]: dispatch 2024-04-22T21:42:06.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 3]}]: dispatch 2024-04-22T21:42:06.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:06.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [4, 2]}]: dispatch 2024-04-22T21:42:06.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.2", "id": [3, 6]}]: dispatch 2024-04-22T21:42:06.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.4", "id": [3, 0]}]: dispatch 2024-04-22T21:42:06.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.11", "id": [3, 6]}]: dispatch 2024-04-22T21:42:06.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.18", "id": [7, 6]}]: dispatch 2024-04-22T21:42:06.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1a", "id": [5, 6]}]: dispatch 2024-04-22T21:42:06.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1e", "id": [5, 1]}]: dispatch 2024-04-22T21:42:06.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1f", "id": [3, 0]}]: dispatch 2024-04-22T21:42:06.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:42:06.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:42:06.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: osdmap e144: 8 total, 7 up, 8 in 2024-04-22T21:42:06.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.9", "id": [7, 2]}]: dispatch 2024-04-22T21:42:06.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 3]}]: dispatch 2024-04-22T21:42:06.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:06.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [4, 2]}]: dispatch 2024-04-22T21:42:06.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.2", "id": [3, 6]}]: dispatch 2024-04-22T21:42:06.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.4", "id": [3, 0]}]: dispatch 2024-04-22T21:42:06.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.11", "id": [3, 6]}]: dispatch 2024-04-22T21:42:06.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.18", "id": [7, 6]}]: dispatch 2024-04-22T21:42:06.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1a", "id": [5, 6]}]: dispatch 2024-04-22T21:42:06.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1e", "id": [5, 1]}]: dispatch 2024-04-22T21:42:06.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1f", "id": [3, 0]}]: dispatch 2024-04-22T21:42:06.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:42:06.348 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:06 smithi150 podman[84813]: 2024-04-22 21:42:06.1308472 +0000 UTC m=+0.653867946 container remove 902636c1a0e44e14dbcee737a30c35922f852d0bc21b328ab7f133bbd4016252 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vendor=Red Hat, Inc., io.openshift.expose-services=, vcs-type=git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, name=centos-stream, release=754, version=8, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T21:42:06.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:42:06.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: osdmap e144: 8 total, 7 up, 8 in 2024-04-22T21:42:06.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.9", "id": [7, 2]}]: dispatch 2024-04-22T21:42:06.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 3]}]: dispatch 2024-04-22T21:42:06.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:06.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [4, 2]}]: dispatch 2024-04-22T21:42:06.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.2", "id": [3, 6]}]: dispatch 2024-04-22T21:42:06.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.4", "id": [3, 0]}]: dispatch 2024-04-22T21:42:06.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.11", "id": [3, 6]}]: dispatch 2024-04-22T21:42:06.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.18", "id": [7, 6]}]: dispatch 2024-04-22T21:42:06.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1a", "id": [5, 6]}]: dispatch 2024-04-22T21:42:06.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1e", "id": [5, 1]}]: dispatch 2024-04-22T21:42:06.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1f", "id": [3, 0]}]: dispatch 2024-04-22T21:42:06.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:42:06.698 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:06 smithi150 podman[84908]: 2024-04-22 21:42:06.366438179 +0000 UTC m=+0.018313052 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:42:06.698 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:06 smithi150 podman[84908]: 2024-04-22 21:42:06.490166721 +0000 UTC m=+0.142041624 container create 410e836c7f0ec1a5f738102597eb52a97e0c4aa09900df128e72861a4953988f (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=main-56e81df, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-22T21:42:07.007 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:06 smithi150 podman[84908]: 2024-04-22 21:42:06.71797832 +0000 UTC m=+0.369853179 container init 410e836c7f0ec1a5f738102597eb52a97e0c4aa09900df128e72861a4953988f (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-22T21:42:07.007 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:06 smithi150 podman[84908]: 2024-04-22 21:42:06.721363605 +0000 UTC m=+0.373238474 container start 410e836c7f0ec1a5f738102597eb52a97e0c4aa09900df128e72861a4953988f (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True) 2024-04-22T21:42:07.007 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:06 smithi150 podman[84908]: 2024-04-22 21:42:06.817782849 +0000 UTC m=+0.469657719 container attach 410e836c7f0ec1a5f738102597eb52a97e0c4aa09900df128e72861a4953988f (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-22T21:42:07.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[82829]: pgmap v214: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 3 op/s 2024-04-22T21:42:07.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.9", "id": [7, 2]}]': finished 2024-04-22T21:42:07.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 3]}]': finished 2024-04-22T21:42:07.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [4, 2]}]': finished 2024-04-22T21:42:07.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.2", "id": [3, 6]}]': finished 2024-04-22T21:42:07.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.4", "id": [3, 0]}]': finished 2024-04-22T21:42:07.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.11", "id": [3, 6]}]': finished 2024-04-22T21:42:07.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.18", "id": [7, 6]}]': finished 2024-04-22T21:42:07.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1a", "id": [5, 6]}]': finished 2024-04-22T21:42:07.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1e", "id": [5, 1]}]': finished 2024-04-22T21:42:07.296 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1f", "id": [3, 0]}]': finished 2024-04-22T21:42:07.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[82829]: osdmap e145: 8 total, 7 up, 8 in 2024-04-22T21:42:07.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[85205]: pgmap v214: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 3 op/s 2024-04-22T21:42:07.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.9", "id": [7, 2]}]': finished 2024-04-22T21:42:07.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 3]}]': finished 2024-04-22T21:42:07.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [4, 2]}]': finished 2024-04-22T21:42:07.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.2", "id": [3, 6]}]': finished 2024-04-22T21:42:07.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.4", "id": [3, 0]}]': finished 2024-04-22T21:42:07.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.11", "id": [3, 6]}]': finished 2024-04-22T21:42:07.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.18", "id": [7, 6]}]': finished 2024-04-22T21:42:07.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1a", "id": [5, 6]}]': finished 2024-04-22T21:42:07.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1e", "id": [5, 1]}]': finished 2024-04-22T21:42:07.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1f", "id": [3, 0]}]': finished 2024-04-22T21:42:07.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:07 smithi130 ceph-mon[85205]: osdmap e145: 8 total, 7 up, 8 in 2024-04-22T21:42:07.321 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:07 smithi150 conmon[84923]: conmon 410e836c7f0ec1a5f738 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-410e836c7f0ec1a5f738102597eb52a97e0c4aa09900df128e72861a4953988f.scope/container/memory.events 2024-04-22T21:42:07.321 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:07 smithi150 podman[84908]: 2024-04-22 21:42:07.028190979 +0000 UTC m=+0.680065846 container died 410e836c7f0ec1a5f738102597eb52a97e0c4aa09900df128e72861a4953988f (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, ceph=True) 2024-04-22T21:42:07.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:07 smithi150 ceph-mon[65103]: pgmap v214: 161 pgs: 13 stale+active+clean, 148 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 3 op/s 2024-04-22T21:42:07.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:07 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.9", "id": [7, 2]}]': finished 2024-04-22T21:42:07.322 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:07 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 3]}]': finished 2024-04-22T21:42:07.322 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:07 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [4, 2]}]': finished 2024-04-22T21:42:07.322 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:07 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.2", "id": [3, 6]}]': finished 2024-04-22T21:42:07.322 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:07 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.4", "id": [3, 0]}]': finished 2024-04-22T21:42:07.322 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:07 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.11", "id": [3, 6]}]': finished 2024-04-22T21:42:07.322 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:07 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.18", "id": [7, 6]}]': finished 2024-04-22T21:42:07.323 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:07 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1a", "id": [5, 6]}]': finished 2024-04-22T21:42:07.323 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:07 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1e", "id": [5, 1]}]': finished 2024-04-22T21:42:07.323 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:07 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "6.1f", "id": [3, 0]}]': finished 2024-04-22T21:42:07.323 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:07 smithi150 ceph-mon[65103]: osdmap e145: 8 total, 7 up, 8 in 2024-04-22T21:42:08.014 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:07 smithi150 podman[84934]: 2024-04-22 21:42:07.622778767 +0000 UTC m=+0.584776429 container remove 410e836c7f0ec1a5f738102597eb52a97e0c4aa09900df128e72861a4953988f (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-56e81df, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-22T21:42:08.014 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:07 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-22T21:42:08.014 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:07 smithi150 systemd[1]: Stopped Ceph osd.6 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:42:08.015 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:07 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.6.service: Consumed 14.812s CPU time. 2024-04-22T21:42:08.015 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:07 smithi150 systemd[1]: Starting Ceph osd.6 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:42:08.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:08 smithi130 ceph-mon[82829]: osdmap e146: 8 total, 7 up, 8 in 2024-04-22T21:42:08.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:08 smithi130 ceph-mon[85205]: osdmap e146: 8 total, 7 up, 8 in 2024-04-22T21:42:08.299 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:08 smithi150 ceph-mon[65103]: osdmap e146: 8 total, 7 up, 8 in 2024-04-22T21:42:08.299 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 podman[85039]: 2024-04-22 21:42:07.967085598 +0000 UTC m=+0.018594560 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:42:08.299 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 podman[85039]: 2024-04-22 21:42:08.107837256 +0000 UTC m=+0.159346217 container create aa3e22ce1f7d86c4e2560a9001243ffc51af2154e065b202ba194b60394cb241 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=main-56e81df, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-22T21:42:08.299 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 podman[85039]: 2024-04-22 21:42:08.318667242 +0000 UTC m=+0.370176202 container init aa3e22ce1f7d86c4e2560a9001243ffc51af2154e065b202ba194b60394cb241 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-56e81df, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-22T21:42:08.598 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 podman[85039]: 2024-04-22 21:42:08.323383766 +0000 UTC m=+0.374892733 container start aa3e22ce1f7d86c4e2560a9001243ffc51af2154e065b202ba194b60394cb241 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, RELEASE=main-56e81df, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:42:08.598 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 podman[85039]: 2024-04-22 21:42:08.454470807 +0000 UTC m=+0.505979770 container attach aa3e22ce1f7d86c4e2560a9001243ffc51af2154e065b202ba194b60394cb241 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df) 2024-04-22T21:42:09.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:09 smithi150 ceph-mon[65103]: pgmap v217: 161 pgs: 1 unknown, 2 peering, 7 active+undersized+degraded, 9 stale+active+clean, 10 active+undersized, 132 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 70 op/s; 29/627 objects degraded (4.625%) 2024-04-22T21:42:09.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:09 smithi150 ceph-mon[65103]: osdmap e147: 8 total, 7 up, 8 in 2024-04-22T21:42:09.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:09 smithi150 ceph-mon[65103]: Health check failed: Degraded data redundancy: 29/627 objects degraded (4.625%), 7 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:09.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:09 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:09.099 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate[85054]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:42:09.099 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate[85054]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:42:09.099 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 bash[85039]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:42:09.100 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 bash[85039]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:42:09.100 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 bash[85039]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T21:42:09.100 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 bash[85039]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T21:42:09.100 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 bash[85039]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T21:42:09.100 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 bash[85039]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T21:42:09.100 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 bash[85039]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-22T21:42:09.100 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 bash[85039]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T21:42:09.100 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 bash[85039]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-22T21:42:09.100 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate[85054]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T21:42:09.101 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate[85054]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T21:42:09.101 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate[85054]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T21:42:09.101 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate[85054]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T21:42:09.101 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate[85054]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-22T21:42:09.101 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate[85054]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T21:42:09.101 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate[85054]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-22T21:42:09.101 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:08 smithi150 podman[85039]: 2024-04-22 21:42:08.829271353 +0000 UTC m=+0.880780315 container died aa3e22ce1f7d86c4e2560a9001243ffc51af2154e065b202ba194b60394cb241 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df, org.label-schema.build-date=20240416) 2024-04-22T21:42:09.294 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:42:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:42:08] "GET /metrics HTTP/1.1" 200 38212 "" "Prometheus/2.43.0" 2024-04-22T21:42:09.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:09 smithi130 ceph-mon[82829]: pgmap v217: 161 pgs: 1 unknown, 2 peering, 7 active+undersized+degraded, 9 stale+active+clean, 10 active+undersized, 132 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 70 op/s; 29/627 objects degraded (4.625%) 2024-04-22T21:42:09.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:09 smithi130 ceph-mon[82829]: osdmap e147: 8 total, 7 up, 8 in 2024-04-22T21:42:09.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:09 smithi130 ceph-mon[82829]: Health check failed: Degraded data redundancy: 29/627 objects degraded (4.625%), 7 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:09.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:09 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:09.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:09 smithi130 ceph-mon[85205]: pgmap v217: 161 pgs: 1 unknown, 2 peering, 7 active+undersized+degraded, 9 stale+active+clean, 10 active+undersized, 132 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 70 op/s; 29/627 objects degraded (4.625%) 2024-04-22T21:42:09.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:09 smithi130 ceph-mon[85205]: osdmap e147: 8 total, 7 up, 8 in 2024-04-22T21:42:09.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:09 smithi130 ceph-mon[85205]: Health check failed: Degraded data redundancy: 29/627 objects degraded (4.625%), 7 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:09.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:09 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:09.753 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:09 smithi150 podman[85182]: 2024-04-22 21:42:09.431319822 +0000 UTC m=+0.596366916 container remove aa3e22ce1f7d86c4e2560a9001243ffc51af2154e065b202ba194b60394cb241 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-56e81df) 2024-04-22T21:42:09.754 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:09 smithi150 podman[85248]: 2024-04-22 21:42:09.648971226 +0000 UTC m=+0.112719763 container create d84338a30cd03ad8437d550f2ac746f8da68674a020a0f21d4e01e06570ba613 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-56e81df) 2024-04-22T21:42:09.754 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:09 smithi150 podman[85248]: 2024-04-22 21:42:09.554253012 +0000 UTC m=+0.018001562 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:42:10.348 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:09 smithi150 podman[85248]: 2024-04-22 21:42:09.876829807 +0000 UTC m=+0.340578352 container init d84338a30cd03ad8437d550f2ac746f8da68674a020a0f21d4e01e06570ba613 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-56e81df) 2024-04-22T21:42:10.349 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:09 smithi150 podman[85248]: 2024-04-22 21:42:09.880197311 +0000 UTC m=+0.343945848 container start d84338a30cd03ad8437d550f2ac746f8da68674a020a0f21d4e01e06570ba613 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6, RELEASE=main-56e81df, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-22T21:42:10.349 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:09 smithi150 ceph-osd[85266]: -- 172.21.15.150:0/3771097732 <== mon.2 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c39a703680 con 0x55c39a6cd400 2024-04-22T21:42:10.349 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:09 smithi150 bash[85248]: d84338a30cd03ad8437d550f2ac746f8da68674a020a0f21d4e01e06570ba613 2024-04-22T21:42:10.349 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:09 smithi150 systemd[1]: Started Ceph osd.6 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:42:10.836 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:42:10.422+0000 7f3231d797c0 -1 Falling back to public interface 2024-04-22T21:42:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:11 smithi150 ceph-mon[65103]: pgmap v219: 161 pgs: 4 active+clean+remapped, 105 active+clean, 27 active+undersized, 1 stale+active+clean, 21 active+undersized+degraded, 2 peering, 1 unknown; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 0 B/s wr, 221 op/s; 78/627 objects degraded (12.440%); 1/627 objects misplaced (0.159%) 2024-04-22T21:42:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:11 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:11 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:11.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:11 smithi130 ceph-mon[82829]: pgmap v219: 161 pgs: 4 active+clean+remapped, 105 active+clean, 27 active+undersized, 1 stale+active+clean, 21 active+undersized+degraded, 2 peering, 1 unknown; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 0 B/s wr, 221 op/s; 78/627 objects degraded (12.440%); 1/627 objects misplaced (0.159%) 2024-04-22T21:42:11.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:11 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:11.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:11 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:11.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:11 smithi130 ceph-mon[85205]: pgmap v219: 161 pgs: 4 active+clean+remapped, 105 active+clean, 27 active+undersized, 1 stale+active+clean, 21 active+undersized+degraded, 2 peering, 1 unknown; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 0 B/s wr, 221 op/s; 78/627 objects degraded (12.440%); 1/627 objects misplaced (0.159%) 2024-04-22T21:42:11.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:11 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:11.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:11 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:13.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:13 smithi130 ceph-mon[82829]: pgmap v220: 161 pgs: 6 active+clean+remapped, 105 active+clean, 27 active+undersized, 21 active+undersized+degraded, 1 peering, 1 unknown; 585 KiB data, 215 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s rd, 0 B/s wr, 210 op/s; 78/627 objects degraded (12.440%); 1/627 objects misplaced (0.159%) 2024-04-22T21:42:13.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:13 smithi130 ceph-mon[85205]: pgmap v220: 161 pgs: 6 active+clean+remapped, 105 active+clean, 27 active+undersized, 21 active+undersized+degraded, 1 peering, 1 unknown; 585 KiB data, 215 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s rd, 0 B/s wr, 210 op/s; 78/627 objects degraded (12.440%); 1/627 objects misplaced (0.159%) 2024-04-22T21:42:13.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:13 smithi150 ceph-mon[65103]: pgmap v220: 161 pgs: 6 active+clean+remapped, 105 active+clean, 27 active+undersized, 21 active+undersized+degraded, 1 peering, 1 unknown; 585 KiB data, 215 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s rd, 0 B/s wr, 210 op/s; 78/627 objects degraded (12.440%); 1/627 objects misplaced (0.159%) 2024-04-22T21:42:13.348 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:42:13.122+0000 7f3231d797c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-04-22T21:42:13.348 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:42:13.147+0000 7f3231d797c0 -1 osd.6 143 log_to_monitors true 2024-04-22T21:42:14.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:14 smithi130 ceph-mon[82829]: from='osd.6 [v2:172.21.15.150:6816/1036674616,v1:172.21.15.150:6817/1036674616]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T21:42:14.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:14 smithi130 ceph-mon[82829]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T21:42:14.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:14 smithi130 ceph-mon[85205]: from='osd.6 [v2:172.21.15.150:6816/1036674616,v1:172.21.15.150:6817/1036674616]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T21:42:14.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:14 smithi130 ceph-mon[85205]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T21:42:14.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:14 smithi150 ceph-mon[65103]: from='osd.6 [v2:172.21.15.150:6816/1036674616,v1:172.21.15.150:6817/1036674616]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T21:42:14.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:14 smithi150 ceph-mon[65103]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T21:42:14.848 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:42:14 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:42:14.519+0000 7f321f1c9700 -1 osd.6 143 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:42:15.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[82829]: pgmap v221: 161 pgs: 9 active+clean+remapped, 104 active+clean, 27 active+undersized, 21 active+undersized+degraded; 585 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 0 B/s wr, 177 op/s; 78/627 objects degraded (12.440%); 1/627 objects misplaced (0.159%) 2024-04-22T21:42:15.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[82829]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T21:42:15.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[82829]: from='osd.6 [v2:172.21.15.150:6816/1036674616,v1:172.21.15.150:6817/1036674616]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:42:15.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[82829]: osdmap e148: 8 total, 7 up, 8 in 2024-04-22T21:42:15.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[82829]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:42:15.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:15.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:15.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[82829]: from='osd.6 ' entity='osd.6' 2024-04-22T21:42:15.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[85205]: pgmap v221: 161 pgs: 9 active+clean+remapped, 104 active+clean, 27 active+undersized, 21 active+undersized+degraded; 585 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 0 B/s wr, 177 op/s; 78/627 objects degraded (12.440%); 1/627 objects misplaced (0.159%) 2024-04-22T21:42:15.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[85205]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T21:42:15.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[85205]: from='osd.6 [v2:172.21.15.150:6816/1036674616,v1:172.21.15.150:6817/1036674616]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:42:15.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[85205]: osdmap e148: 8 total, 7 up, 8 in 2024-04-22T21:42:15.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[85205]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:42:15.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:15.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:15.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:15 smithi130 ceph-mon[85205]: from='osd.6 ' entity='osd.6' 2024-04-22T21:42:15.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:15 smithi150 ceph-mon[65103]: pgmap v221: 161 pgs: 9 active+clean+remapped, 104 active+clean, 27 active+undersized, 21 active+undersized+degraded; 585 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 0 B/s wr, 177 op/s; 78/627 objects degraded (12.440%); 1/627 objects misplaced (0.159%) 2024-04-22T21:42:15.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:15 smithi150 ceph-mon[65103]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T21:42:15.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:15 smithi150 ceph-mon[65103]: from='osd.6 [v2:172.21.15.150:6816/1036674616,v1:172.21.15.150:6817/1036674616]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:42:15.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:15 smithi150 ceph-mon[65103]: osdmap e148: 8 total, 7 up, 8 in 2024-04-22T21:42:15.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:15 smithi150 ceph-mon[65103]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:42:15.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:15 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:15.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:15 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:15.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:15 smithi150 ceph-mon[65103]: from='osd.6 ' entity='osd.6' 2024-04-22T21:42:16.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:16 smithi130 ceph-mon[82829]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:42:16.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:16 smithi130 ceph-mon[82829]: osd.6 [v2:172.21.15.150:6816/1036674616,v1:172.21.15.150:6817/1036674616] boot 2024-04-22T21:42:16.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:16 smithi130 ceph-mon[82829]: osdmap e149: 8 total, 8 up, 8 in 2024-04-22T21:42:16.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:16 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:42:16.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:16 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:16.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:16 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:16.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:16 smithi130 ceph-mon[85205]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:42:16.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:16 smithi130 ceph-mon[85205]: osd.6 [v2:172.21.15.150:6816/1036674616,v1:172.21.15.150:6817/1036674616] boot 2024-04-22T21:42:16.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:16 smithi130 ceph-mon[85205]: osdmap e149: 8 total, 8 up, 8 in 2024-04-22T21:42:16.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:16 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:42:16.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:16 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:16.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:16 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:16.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:16 smithi150 ceph-mon[65103]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:42:16.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:16 smithi150 ceph-mon[65103]: osd.6 [v2:172.21.15.150:6816/1036674616,v1:172.21.15.150:6817/1036674616] boot 2024-04-22T21:42:16.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:16 smithi150 ceph-mon[65103]: osdmap e149: 8 total, 8 up, 8 in 2024-04-22T21:42:16.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:16 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T21:42:16.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:16 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:16.349 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:16 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:17.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:17 smithi130 ceph-mon[82829]: pgmap v224: 161 pgs: 9 active+clean+remapped, 104 active+clean, 27 active+undersized, 21 active+undersized+degraded; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 0 B/s wr, 127 op/s; 78/627 objects degraded (12.440%); 1/627 objects misplaced (0.159%) 2024-04-22T21:42:17.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:17 smithi130 ceph-mon[82829]: osdmap e150: 8 total, 8 up, 8 in 2024-04-22T21:42:17.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:17 smithi130 ceph-mon[82829]: Health check update: Degraded data redundancy: 78/627 objects degraded (12.440%), 21 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:17.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:17 smithi130 ceph-mon[85205]: pgmap v224: 161 pgs: 9 active+clean+remapped, 104 active+clean, 27 active+undersized, 21 active+undersized+degraded; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 0 B/s wr, 127 op/s; 78/627 objects degraded (12.440%); 1/627 objects misplaced (0.159%) 2024-04-22T21:42:17.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:17 smithi130 ceph-mon[85205]: osdmap e150: 8 total, 8 up, 8 in 2024-04-22T21:42:17.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:17 smithi130 ceph-mon[85205]: Health check update: Degraded data redundancy: 78/627 objects degraded (12.440%), 21 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:17.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:17 smithi150 ceph-mon[65103]: pgmap v224: 161 pgs: 9 active+clean+remapped, 104 active+clean, 27 active+undersized, 21 active+undersized+degraded; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 0 B/s wr, 127 op/s; 78/627 objects degraded (12.440%); 1/627 objects misplaced (0.159%) 2024-04-22T21:42:17.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:17 smithi150 ceph-mon[65103]: osdmap e150: 8 total, 8 up, 8 in 2024-04-22T21:42:17.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:17 smithi150 ceph-mon[65103]: Health check update: Degraded data redundancy: 78/627 objects degraded (12.440%), 21 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:18.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:18 smithi130 ceph-mon[82829]: osdmap e151: 8 total, 8 up, 8 in 2024-04-22T21:42:18.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:18 smithi130 ceph-mon[85205]: osdmap e151: 8 total, 8 up, 8 in 2024-04-22T21:42:18.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:18 smithi150 ceph-mon[65103]: osdmap e151: 8 total, 8 up, 8 in 2024-04-22T21:42:19.294 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:42:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:42:18] "GET /metrics HTTP/1.1" 200 38212 "" "Prometheus/2.43.0" 2024-04-22T21:42:19.294 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:42:19.133+0000 7fd7212f0700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-22T21:42:19.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[85205]: pgmap v227: 161 pgs: 2 peering, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 7 active+clean+remapped, 115 active+clean, 19 active+undersized, 16 active+undersized+degraded; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 59/627 objects degraded (9.410%); 1/627 objects misplaced (0.159%); 0 B/s, 2 objects/s recovering 2024-04-22T21:42:19.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:19.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:19.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:19.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:19.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:42:19.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:19.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:42:19.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:19.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:19.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:19.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T21:42:19.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[82829]: pgmap v227: 161 pgs: 2 peering, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 7 active+clean+remapped, 115 active+clean, 19 active+undersized, 16 active+undersized+degraded; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 59/627 objects degraded (9.410%); 1/627 objects misplaced (0.159%); 0 B/s, 2 objects/s recovering 2024-04-22T21:42:19.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:19.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:19.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:19.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:19.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:42:19.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:19.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:42:19.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:19.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:19.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:19.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:19 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T21:42:19.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:19 smithi150 ceph-mon[65103]: pgmap v227: 161 pgs: 2 peering, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 7 active+clean+remapped, 115 active+clean, 19 active+undersized, 16 active+undersized+degraded; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 59/627 objects degraded (9.410%); 1/627 objects misplaced (0.159%); 0 B/s, 2 objects/s recovering 2024-04-22T21:42:19.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:19 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:19.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:19 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:19.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:19 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:19.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:19 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:19.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:19 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:42:19.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:19 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:19.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:19 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:42:19.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:19 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:19.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:19 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:19.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:19 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:19.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:19 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T21:42:20.767 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:20 smithi150 ceph-mon[65103]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T21:42:20.768 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:20 smithi150 ceph-mon[65103]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-22T21:42:20.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:20 smithi130 ceph-mon[85205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T21:42:20.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:20 smithi130 ceph-mon[85205]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-22T21:42:20.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:20 smithi130 ceph-mon[82829]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T21:42:20.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:20 smithi130 ceph-mon[82829]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-22T21:42:21.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:21 smithi150 ceph-mon[65103]: pgmap v228: 161 pgs: 2 peering, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 1 active+clean+remapped, 156 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 718 B/s rd, 0 op/s; 7/627 objects degraded (1.116%); 3 B/s, 8 objects/s recovering 2024-04-22T21:42:21.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:21.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:42:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:21 smithi130 ceph-mon[85205]: pgmap v228: 161 pgs: 2 peering, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 1 active+clean+remapped, 156 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 718 B/s rd, 0 op/s; 7/627 objects degraded (1.116%); 3 B/s, 8 objects/s recovering 2024-04-22T21:42:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:42:22.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:21 smithi130 ceph-mon[82829]: pgmap v228: 161 pgs: 2 peering, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 1 active+clean+remapped, 156 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 718 B/s rd, 0 op/s; 7/627 objects degraded (1.116%); 3 B/s, 8 objects/s recovering 2024-04-22T21:42:22.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:22.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:42:23.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:22 smithi130 ceph-mon[82829]: Health check update: Degraded data redundancy: 7/627 objects degraded (1.116%), 2 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:23.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:22 smithi130 ceph-mon[85205]: Health check update: Degraded data redundancy: 7/627 objects degraded (1.116%), 2 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:23.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:22 smithi150 ceph-mon[65103]: Health check update: Degraded data redundancy: 7/627 objects degraded (1.116%), 2 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:23 smithi130 ceph-mon[85205]: pgmap v229: 161 pgs: 2 peering, 159 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 657 B/s rd, 0 op/s; 3 B/s, 8 objects/s recovering 2024-04-22T21:42:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:23 smithi130 ceph-mon[85205]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/627 objects degraded (1.116%), 2 pgs degraded) 2024-04-22T21:42:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:23 smithi130 ceph-mon[85205]: Cluster is now healthy 2024-04-22T21:42:24.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:23 smithi130 ceph-mon[82829]: pgmap v229: 161 pgs: 2 peering, 159 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 657 B/s rd, 0 op/s; 3 B/s, 8 objects/s recovering 2024-04-22T21:42:24.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:23 smithi130 ceph-mon[82829]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/627 objects degraded (1.116%), 2 pgs degraded) 2024-04-22T21:42:24.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:23 smithi130 ceph-mon[82829]: Cluster is now healthy 2024-04-22T21:42:24.086 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:42:24.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:23 smithi150 ceph-mon[65103]: pgmap v229: 161 pgs: 2 peering, 159 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 657 B/s rd, 0 op/s; 3 B/s, 8 objects/s recovering 2024-04-22T21:42:24.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:23 smithi150 ceph-mon[65103]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/627 objects degraded (1.116%), 2 pgs degraded) 2024-04-22T21:42:24.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:23 smithi150 ceph-mon[65103]: Cluster is now healthy 2024-04-22T21:42:24.743 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:42:24.743 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (8m) 114s ago 31m 27.7M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:42:24.743 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (8m) 10s ago 30m 83.0M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:42:24.743 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (8m) 114s ago 30m 60.5M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:42:24.743 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (10m) 10s ago 35m 454M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:42:24.743 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (19m) 114s ago 37m 510M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:42:24.743 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (6m) 114s ago 37m 55.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (7m) 10s ago 36m 48.7M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (6m) 114s ago 36m 41.2M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (9m) 114s ago 31m 15.7M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (9m) 10s ago 31m 15.2M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (4m) 114s ago 35m 47.6M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (3m) 114s ago 35m 52.4M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (2m) 114s ago 34m 41.5M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 027e1ac8c177 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (119s) 114s ago 34m 12.1M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 ca6c16e44590 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (85s) 10s ago 33m 53.5M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 0c80f1b448cb 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (50s) 10s ago 33m 47.1M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 aef1942f3f6b 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (15s) 10s ago 33m 11.8M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 d84338a30cd0 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (32m) 10s ago 32m 61.2M 3248M 17.2.0 e1d6a67b021e 69f17767728c 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (9m) 10s ago 31m 74.0M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (30m) 114s ago 30m 80.2M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:42:24.744 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (30m) 10s ago 30m 78.3M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:42:25.142 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:42:25.142 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:42:25.142 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:42:25.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:42:25.142 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:42:25.142 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:42:25.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:42:25.142 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:42:25.142 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T21:42:25.142 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 7 2024-04-22T21:42:25.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:42:25.143 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:42:25.143 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:42:25.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:42:25.143 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:42:25.143 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-22T21:42:25.143 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 12 2024-04-22T21:42:25.143 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:42:25.143 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:42:25.476 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:42:25.476 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:42:25.476 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:42:25.476 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-22T21:42:25.476 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:42:25.476 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "7/8 daemons upgraded", 2024-04-22T21:42:25.476 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T21:42:25.476 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:42:25.477 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:42:25.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:25 smithi130 ceph-mon[82829]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 2 B/s, 6 objects/s recovering 2024-04-22T21:42:25.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:25 smithi130 ceph-mon[82829]: from='client.54573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:25.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:25 smithi130 ceph-mon[82829]: from='client.54579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:25.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:25 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/3225459561' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:25.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:25 smithi130 ceph-mon[85205]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 2 B/s, 6 objects/s recovering 2024-04-22T21:42:25.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:25 smithi130 ceph-mon[85205]: from='client.54573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:25.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:25 smithi130 ceph-mon[85205]: from='client.54579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:25.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:25 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/3225459561' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:26.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:25 smithi150 ceph-mon[65103]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 2 B/s, 6 objects/s recovering 2024-04-22T21:42:26.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:25 smithi150 ceph-mon[65103]: from='client.54573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:26.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:25 smithi150 ceph-mon[65103]: from='client.54579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:26.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:25 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/3225459561' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:27.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:26 smithi130 ceph-mon[82829]: from='client.54585 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:27.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:26 smithi130 ceph-mon[82829]: from='client.64531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:27.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:26 smithi130 ceph-mon[85205]: from='client.54585 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:27.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:26 smithi130 ceph-mon[85205]: from='client.64531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:27.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:26 smithi150 ceph-mon[65103]: from='client.54585 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:27.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:26 smithi150 ceph-mon[65103]: from='client.64531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:28.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:27 smithi130 ceph-mon[82829]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s; 2 B/s, 6 objects/s recovering 2024-04-22T21:42:28.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:27 smithi130 ceph-mon[85205]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s; 2 B/s, 6 objects/s recovering 2024-04-22T21:42:28.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:27 smithi150 ceph-mon[65103]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s; 2 B/s, 6 objects/s recovering 2024-04-22T21:42:29.297 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:42:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:42:28] "GET /metrics HTTP/1.1" 200 38225 "" "Prometheus/2.43.0" 2024-04-22T21:42:30.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:29 smithi130 ceph-mon[85205]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s; 1 B/s, 4 objects/s recovering 2024-04-22T21:42:30.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:29 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:30.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:29 smithi130 ceph-mon[82829]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s; 1 B/s, 4 objects/s recovering 2024-04-22T21:42:30.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:29 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:30.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:29 smithi150 ceph-mon[65103]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s; 1 B/s, 4 objects/s recovering 2024-04-22T21:42:30.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:29 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:30 smithi130 ceph-mon[85205]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 1 B/s, 3 objects/s recovering 2024-04-22T21:42:31.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:30 smithi130 ceph-mon[82829]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 1 B/s, 3 objects/s recovering 2024-04-22T21:42:31.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:30 smithi150 ceph-mon[65103]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 1 B/s, 3 objects/s recovering 2024-04-22T21:42:32.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:32 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:32.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:32 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:32.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:32 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:33.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:33 smithi150 ceph-mon[65103]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-04-22T21:42:33.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:33 smithi130 ceph-mon[85205]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-04-22T21:42:33.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:33 smithi130 ceph-mon[82829]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 0 B/s, 0 objects/s recovering 2024-04-22T21:42:35.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:35 smithi150 ceph-mon[65103]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:35.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:35 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T21:42:35.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:35 smithi150 ceph-mon[65103]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T21:42:35.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:35 smithi150 ceph-mon[65103]: Upgrade: osd.7 is safe to restart 2024-04-22T21:42:35.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:35 smithi130 ceph-mon[85205]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:35.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:35 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T21:42:35.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:35 smithi130 ceph-mon[85205]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T21:42:35.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:35 smithi130 ceph-mon[85205]: Upgrade: osd.7 is safe to restart 2024-04-22T21:42:35.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:35 smithi130 ceph-mon[82829]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:35.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:35 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T21:42:35.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:35 smithi130 ceph-mon[82829]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T21:42:35.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:35 smithi130 ceph-mon[82829]: Upgrade: osd.7 is safe to restart 2024-04-22T21:42:36.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:36 smithi150 ceph-mon[65103]: Upgrade: Updating osd.7 2024-04-22T21:42:36.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:36.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T21:42:36.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:36.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:36 smithi150 ceph-mon[65103]: Deploying daemon osd.7 on smithi150 2024-04-22T21:42:36.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:36.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:42:37.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[82829]: Upgrade: Updating osd.7 2024-04-22T21:42:37.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:37.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T21:42:37.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:37.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[82829]: Deploying daemon osd.7 on smithi150 2024-04-22T21:42:37.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:37.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:42:37.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[85205]: Upgrade: Updating osd.7 2024-04-22T21:42:37.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:37.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T21:42:37.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:37.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[85205]: Deploying daemon osd.7 on smithi150 2024-04-22T21:42:37.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:37.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:42:38.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:37 smithi130 ceph-mon[85205]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:42:38.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:37 smithi130 ceph-mon[82829]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:42:38.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:37 smithi150 ceph-mon[65103]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:42:39.098 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:38 smithi150 systemd[1]: Stopping Ceph osd.7 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:42:39.294 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:42:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:42:38] "GET /metrics HTTP/1.1" 200 38221 "" "Prometheus/2.43.0" 2024-04-22T21:42:39.598 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[34862]: 2024-04-22T21:42:39.121+0000 7fc5314a8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:42:39.599 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[34862]: 2024-04-22T21:42:39.121+0000 7fc5314a8700 -1 osd.7 151 *** Got signal Terminated *** 2024-04-22T21:42:39.599 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:39 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[34862]: 2024-04-22T21:42:39.121+0000 7fc5314a8700 -1 osd.7 151 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:42:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:39 smithi130 ceph-mon[85205]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:39 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:39 smithi130 ceph-mon[85205]: osd.7 marked itself down and dead 2024-04-22T21:42:40.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:39 smithi130 ceph-mon[82829]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:40.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:39 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:40.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:39 smithi130 ceph-mon[82829]: osd.7 marked itself down and dead 2024-04-22T21:42:40.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:39 smithi150 ceph-mon[65103]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:40.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:39 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:40.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:39 smithi150 ceph-mon[65103]: osd.7 marked itself down and dead 2024-04-22T21:42:40.413 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:40 smithi150 podman[89491]: 2024-04-22 21:42:40.169108672 +0000 UTC m=+1.180376713 container died 69f17767728cd3dc0dae8cc43e5bf88ee64c9911997f90ce304919eb21a760f0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, release=754, distribution-scope=public, version=8, name=centos-stream, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, architecture=x86_64, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-22T21:42:40.732 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:40 smithi150 podman[89491]: 2024-04-22 21:42:40.530790091 +0000 UTC m=+1.542058129 container cleanup 69f17767728cd3dc0dae8cc43e5bf88ee64c9911997f90ce304919eb21a760f0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, version=8, vendor=Red Hat, Inc., GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, ceph=True, release=754) 2024-04-22T21:42:40.732 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:40 smithi150 bash[89491]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7 2024-04-22T21:42:41.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:40 smithi130 ceph-mon[85205]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:42:41.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:40 smithi130 ceph-mon[85205]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-22T21:42:41.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:40 smithi130 ceph-mon[85205]: osdmap e152: 8 total, 7 up, 8 in 2024-04-22T21:42:41.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:40 smithi130 ceph-mon[85205]: osdmap e153: 8 total, 7 up, 8 in 2024-04-22T21:42:41.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:40 smithi130 ceph-mon[82829]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:42:41.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:40 smithi130 ceph-mon[82829]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-22T21:42:41.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:40 smithi130 ceph-mon[82829]: osdmap e152: 8 total, 7 up, 8 in 2024-04-22T21:42:41.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:40 smithi130 ceph-mon[82829]: osdmap e153: 8 total, 7 up, 8 in 2024-04-22T21:42:41.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:40 smithi150 ceph-mon[65103]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T21:42:41.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:40 smithi150 ceph-mon[65103]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-22T21:42:41.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:40 smithi150 ceph-mon[65103]: osdmap e152: 8 total, 7 up, 8 in 2024-04-22T21:42:41.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:40 smithi150 ceph-mon[65103]: osdmap e153: 8 total, 7 up, 8 in 2024-04-22T21:42:41.098 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:40 smithi150 podman[89518]: 2024-04-22 21:42:40.815673686 +0000 UTC m=+0.641507113 container remove 69f17767728cd3dc0dae8cc43e5bf88ee64c9911997f90ce304919eb21a760f0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, release=754, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, version=8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , architecture=x86_64, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870) 2024-04-22T21:42:41.416 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:41 smithi150 podman[89608]: 2024-04-22 21:42:41.047083881 +0000 UTC m=+0.018519249 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:42:41.416 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:41 smithi150 podman[89608]: 2024-04-22 21:42:41.149795771 +0000 UTC m=+0.121231157 container create 2724fda40408b9a8518a5be85bced41722070e13ce57b65e3606b317830c75c9 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-22T21:42:41.416 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:41 smithi150 podman[89608]: 2024-04-22 21:42:41.352222997 +0000 UTC m=+0.323658341 container init 2724fda40408b9a8518a5be85bced41722070e13ce57b65e3606b317830c75c9 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , RELEASE=main-56e81df) 2024-04-22T21:42:41.416 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:41 smithi150 podman[89608]: 2024-04-22 21:42:41.355270925 +0000 UTC m=+0.326706276 container start 2724fda40408b9a8518a5be85bced41722070e13ce57b65e3606b317830c75c9 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, RELEASE=main-56e81df, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-22T21:42:41.416 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:41 smithi150 podman[89608]: 2024-04-22 21:42:41.435485158 +0000 UTC m=+0.406920518 container attach 2724fda40408b9a8518a5be85bced41722070e13ce57b65e3606b317830c75c9 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, RELEASE=main-56e81df, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-22T21:42:41.733 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:41 smithi150 podman[89608]: 2024-04-22 21:42:41.6627168 +0000 UTC m=+0.634152157 container died 2724fda40408b9a8518a5be85bced41722070e13ce57b65e3606b317830c75c9 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=main-56e81df, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:42:41.986 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:41 smithi150 ceph-mon[65103]: pgmap v239: 161 pgs: 22 stale+active+clean, 139 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:42:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:41 smithi130 ceph-mon[85205]: pgmap v239: 161 pgs: 22 stale+active+clean, 139 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:42:42.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:41 smithi130 ceph-mon[82829]: pgmap v239: 161 pgs: 22 stale+active+clean, 139 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:42:42.598 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:42 smithi150 podman[89634]: 2024-04-22 21:42:42.306234776 +0000 UTC m=+0.638165343 container remove 2724fda40408b9a8518a5be85bced41722070e13ce57b65e3606b317830c75c9 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=main-56e81df, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240416) 2024-04-22T21:42:42.598 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:42 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.7.service: Deactivated successfully. 2024-04-22T21:42:42.598 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:42 smithi150 systemd[1]: Stopped Ceph osd.7 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:42:42.598 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:42 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.7.service: Consumed 18.606s CPU time. 2024-04-22T21:42:42.598 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:42 smithi150 systemd[1]: Starting Ceph osd.7 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:42:42.991 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:42 smithi150 ceph-mon[65103]: pgmap v241: 161 pgs: 3 active+undersized+degraded, 2 active+undersized, 7 peering, 18 stale+active+clean, 131 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 3/627 objects degraded (0.478%) 2024-04-22T21:42:42.991 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:42 smithi150 podman[89739]: 2024-04-22 21:42:42.658212488 +0000 UTC m=+0.018363366 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:42:42.991 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:42 smithi150 podman[89739]: 2024-04-22 21:42:42.7576107 +0000 UTC m=+0.117761578 container create d80bab60b993536d8b4e9af2f10135ce24f43a0cca1c5fe3e49355ce45a67a60 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-56e81df, maintainer=Guillaume Abrioux ) 2024-04-22T21:42:42.991 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 podman[89739]: 2024-04-22 21:42:43.010781432 +0000 UTC m=+0.370932319 container init d80bab60b993536d8b4e9af2f10135ce24f43a0cca1c5fe3e49355ce45a67a60 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416) 2024-04-22T21:42:43.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:42 smithi130 ceph-mon[85205]: pgmap v241: 161 pgs: 3 active+undersized+degraded, 2 active+undersized, 7 peering, 18 stale+active+clean, 131 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 3/627 objects degraded (0.478%) 2024-04-22T21:42:43.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:42 smithi130 ceph-mon[82829]: pgmap v241: 161 pgs: 3 active+undersized+degraded, 2 active+undersized, 7 peering, 18 stale+active+clean, 131 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 3/627 objects degraded (0.478%) 2024-04-22T21:42:43.348 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 podman[89739]: 2024-04-22 21:42:43.014236134 +0000 UTC m=+0.374387023 container start d80bab60b993536d8b4e9af2f10135ce24f43a0cca1c5fe3e49355ce45a67a60 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_CLEAN=True) 2024-04-22T21:42:43.348 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 podman[89739]: 2024-04-22 21:42:43.118520318 +0000 UTC m=+0.478671206 container attach d80bab60b993536d8b4e9af2f10135ce24f43a0cca1c5fe3e49355ce45a67a60 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=main-56e81df, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T21:42:43.749 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate[89754]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:42:43.749 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate[89754]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:42:43.749 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 bash[89739]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:42:43.749 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 bash[89739]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T21:42:43.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 bash[89739]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T21:42:43.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 bash[89739]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T21:42:43.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 bash[89739]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T21:42:43.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 bash[89739]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T21:42:43.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 bash[89739]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-22T21:42:43.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 bash[89739]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T21:42:43.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 bash[89739]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-22T21:42:43.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate[89754]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T21:42:43.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate[89754]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T21:42:43.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate[89754]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T21:42:43.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate[89754]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T21:42:43.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate[89754]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-22T21:42:43.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate[89754]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T21:42:43.751 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate[89754]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-22T21:42:43.751 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:43 smithi150 podman[89739]: 2024-04-22 21:42:43.521443164 +0000 UTC m=+0.881594043 container died d80bab60b993536d8b4e9af2f10135ce24f43a0cca1c5fe3e49355ce45a67a60 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=) 2024-04-22T21:42:44.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:43 smithi130 ceph-mon[82829]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-22T21:42:44.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:43 smithi130 ceph-mon[82829]: Health check failed: Degraded data redundancy: 3/627 objects degraded (0.478%), 3 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:43 smithi130 ceph-mon[85205]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-22T21:42:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:43 smithi130 ceph-mon[85205]: Health check failed: Degraded data redundancy: 3/627 objects degraded (0.478%), 3 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:44.077 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:43 smithi150 ceph-mon[65103]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-22T21:42:44.078 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:43 smithi150 ceph-mon[65103]: Health check failed: Degraded data redundancy: 3/627 objects degraded (0.478%), 3 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:44.337 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:44 smithi150 podman[89882]: 2024-04-22 21:42:44.097622718 +0000 UTC m=+0.566873554 container remove d80bab60b993536d8b4e9af2f10135ce24f43a0cca1c5fe3e49355ce45a67a60 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T21:42:44.337 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:44 smithi150 podman[89947]: 2024-04-22 21:42:44.214009294 +0000 UTC m=+0.018613272 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:42:44.338 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:44 smithi150 podman[89947]: 2024-04-22 21:42:44.357142674 +0000 UTC m=+0.161746690 container create d0d3f15f9e88f54c809e71fc5c81880c28a7b2dedef219960434273e18fbe55c (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=main-56e81df) 2024-04-22T21:42:44.598 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:44 smithi150 podman[89947]: 2024-04-22 21:42:44.584086146 +0000 UTC m=+0.388690118 container init d0d3f15f9e88f54c809e71fc5c81880c28a7b2dedef219960434273e18fbe55c (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7, GIT_BRANCH=HEAD, RELEASE=main-56e81df, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:42:44.598 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:44 smithi150 podman[89947]: 2024-04-22 21:42:44.587055619 +0000 UTC m=+0.391659589 container start d0d3f15f9e88f54c809e71fc5c81880c28a7b2dedef219960434273e18fbe55c (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7, RELEASE=main-56e81df, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T21:42:44.598 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:44 smithi150 ceph-osd[89965]: -- 172.21.15.150:0/3990560029 <== mon.2 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560e52d3f680 con 0x560e52d09400 2024-04-22T21:42:45.098 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:44 smithi150 bash[89947]: d0d3f15f9e88f54c809e71fc5c81880c28a7b2dedef219960434273e18fbe55c 2024-04-22T21:42:45.098 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:44 smithi150 systemd[1]: Started Ceph osd.7 for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:42:45.528 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:45 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:42:45.124+0000 7f0786bee7c0 -1 Falling back to public interface 2024-04-22T21:42:45.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:45 smithi150 ceph-mon[65103]: pgmap v242: 161 pgs: 15 active+undersized+degraded, 12 active+undersized, 7 peering, 8 stale+active+clean, 119 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 47/627 objects degraded (7.496%) 2024-04-22T21:42:45.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:45 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:45.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:45 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:46.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:45 smithi130 ceph-mon[82829]: pgmap v242: 161 pgs: 15 active+undersized+degraded, 12 active+undersized, 7 peering, 8 stale+active+clean, 119 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 47/627 objects degraded (7.496%) 2024-04-22T21:42:46.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:45 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:46.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:45 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:46.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:45 smithi130 ceph-mon[85205]: pgmap v242: 161 pgs: 15 active+undersized+degraded, 12 active+undersized, 7 peering, 8 stale+active+clean, 119 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 47/627 objects degraded (7.496%) 2024-04-22T21:42:46.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:45 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:46.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:45 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:47.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:46 smithi130 ceph-mon[82829]: pgmap v243: 161 pgs: 28 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T21:42:47.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:46 smithi130 ceph-mon[82829]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-04-22T21:42:47.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:46 smithi130 ceph-mon[85205]: pgmap v243: 161 pgs: 28 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T21:42:47.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:46 smithi130 ceph-mon[85205]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-04-22T21:42:47.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:46 smithi150 ceph-mon[65103]: pgmap v243: 161 pgs: 28 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T21:42:47.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:46 smithi150 ceph-mon[65103]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-04-22T21:42:48.098 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:47 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:42:47.821+0000 7f0786bee7c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-04-22T21:42:48.098 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:47 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:42:47.851+0000 7f0786bee7c0 -1 osd.7 151 log_to_monitors true 2024-04-22T21:42:48.999 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:48 smithi150 ceph-mon[65103]: pgmap v244: 161 pgs: 28 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T21:42:48.999 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:48 smithi150 ceph-mon[65103]: from='osd.7 [v2:172.21.15.150:6824/3915700177,v1:172.21.15.150:6825/3915700177]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T21:42:48.999 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:48 smithi150 ceph-mon[65103]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T21:42:49.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:48 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:49.044 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:42:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:42:48] "GET /metrics HTTP/1.1" 200 38221 "" "Prometheus/2.43.0" 2024-04-22T21:42:49.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:48 smithi130 ceph-mon[82829]: pgmap v244: 161 pgs: 28 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T21:42:49.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:48 smithi130 ceph-mon[82829]: from='osd.7 [v2:172.21.15.150:6824/3915700177,v1:172.21.15.150:6825/3915700177]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T21:42:49.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:48 smithi130 ceph-mon[82829]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T21:42:49.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:48 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:49.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:48 smithi130 ceph-mon[85205]: pgmap v244: 161 pgs: 28 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 239 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T21:42:49.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:48 smithi130 ceph-mon[85205]: from='osd.7 [v2:172.21.15.150:6824/3915700177,v1:172.21.15.150:6825/3915700177]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T21:42:49.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:48 smithi130 ceph-mon[85205]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T21:42:49.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:48 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:49.598 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:42:49 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:42:49.280+0000 7f077403e700 -1 osd.7 151 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T21:42:50.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[82829]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T21:42:50.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[82829]: osdmap e154: 8 total, 7 up, 8 in 2024-04-22T21:42:50.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[82829]: from='osd.7 [v2:172.21.15.150:6824/3915700177,v1:172.21.15.150:6825/3915700177]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:42:50.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[82829]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:42:50.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:50.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:50.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[82829]: from='osd.7 ' entity='osd.7' 2024-04-22T21:42:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[85205]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T21:42:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[85205]: osdmap e154: 8 total, 7 up, 8 in 2024-04-22T21:42:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[85205]: from='osd.7 [v2:172.21.15.150:6824/3915700177,v1:172.21.15.150:6825/3915700177]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:42:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[85205]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:42:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:49 smithi130 ceph-mon[85205]: from='osd.7 ' entity='osd.7' 2024-04-22T21:42:50.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:49 smithi150 ceph-mon[65103]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T21:42:50.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:49 smithi150 ceph-mon[65103]: osdmap e154: 8 total, 7 up, 8 in 2024-04-22T21:42:50.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:49 smithi150 ceph-mon[65103]: from='osd.7 [v2:172.21.15.150:6824/3915700177,v1:172.21.15.150:6825/3915700177]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:42:50.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:49 smithi150 ceph-mon[65103]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-22T21:42:50.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:49 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:50.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:49 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:50.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:49 smithi150 ceph-mon[65103]: from='osd.7 ' entity='osd.7' 2024-04-22T21:42:51.508 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:51 smithi150 ceph-mon[65103]: pgmap v246: 161 pgs: 28 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T21:42:51.508 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:51.508 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:51.508 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:51 smithi150 ceph-mon[65103]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:42:51.508 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:51 smithi150 ceph-mon[65103]: osd.7 [v2:172.21.15.150:6824/3915700177,v1:172.21.15.150:6825/3915700177] boot 2024-04-22T21:42:51.508 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:51 smithi150 ceph-mon[65103]: osdmap e155: 8 total, 8 up, 8 in 2024-04-22T21:42:51.508 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:42:51.508 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:51.508 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:42:51.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[85205]: pgmap v246: 161 pgs: 28 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T21:42:51.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:51.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:51.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[85205]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:42:51.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[85205]: osd.7 [v2:172.21.15.150:6824/3915700177,v1:172.21.15.150:6825/3915700177] boot 2024-04-22T21:42:51.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[85205]: osdmap e155: 8 total, 8 up, 8 in 2024-04-22T21:42:51.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:42:51.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:51.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:42:51.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[82829]: pgmap v246: 161 pgs: 28 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T21:42:51.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:51.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:51.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[82829]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T21:42:51.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[82829]: osd.7 [v2:172.21.15.150:6824/3915700177,v1:172.21.15.150:6825/3915700177] boot 2024-04-22T21:42:51.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[82829]: osdmap e155: 8 total, 8 up, 8 in 2024-04-22T21:42:51.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T21:42:51.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:51.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:42:52.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:52 smithi150 ceph-mon[65103]: osdmap e156: 8 total, 8 up, 8 in 2024-04-22T21:42:52.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:52 smithi150 ceph-mon[65103]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:52.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:52 smithi130 ceph-mon[82829]: osdmap e156: 8 total, 8 up, 8 in 2024-04-22T21:42:52.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:52 smithi130 ceph-mon[82829]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:52.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:52 smithi130 ceph-mon[85205]: osdmap e156: 8 total, 8 up, 8 in 2024-04-22T21:42:52.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:52 smithi130 ceph-mon[85205]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 28 pgs degraded (PG_DEGRADED) 2024-04-22T21:42:53.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:53 smithi130 ceph-mon[85205]: pgmap v249: 161 pgs: 25 active+undersized+degraded, 38 active+undersized, 98 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 80/627 objects degraded (12.759%) 2024-04-22T21:42:53.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:53 smithi130 ceph-mon[82829]: pgmap v249: 161 pgs: 25 active+undersized+degraded, 38 active+undersized, 98 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 80/627 objects degraded (12.759%) 2024-04-22T21:42:53.565 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:53 smithi150 ceph-mon[65103]: pgmap v249: 161 pgs: 25 active+undersized+degraded, 38 active+undersized, 98 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 80/627 objects degraded (12.759%) 2024-04-22T21:42:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:42:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-22T21:42:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-22T21:42:55.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-22T21:42:55.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-22T21:42:55.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-22T21:42:55.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-22T21:42:55.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-22T21:42:55.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-22T21:42:55.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-22T21:42:55.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-22T21:42:55.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-22T21:42:55.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-22T21:42:55.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-22T21:42:55.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-22T21:42:55.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-22T21:42:55.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-22T21:42:55.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-22T21:42:55.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-22T21:42:55.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-22T21:42:55.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-22T21:42:55.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-22T21:42:55.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-22T21:42:55.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-22T21:42:55.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-22T21:42:55.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:54 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-22T21:42:55.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:55.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:55.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:55.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:42:55.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:55.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:42:55.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-22T21:42:55.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-22T21:42:55.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-22T21:42:55.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-22T21:42:55.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-22T21:42:55.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-22T21:42:55.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-22T21:42:55.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-22T21:42:55.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:54 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-22T21:42:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: pgmap v250: 161 pgs: 19 active+undersized+degraded, 32 active+undersized, 110 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 62/627 objects degraded (9.888%) 2024-04-22T21:42:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all osd 2024-04-22T21:42:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: Upgrade: Setting require_osd_release to 19 squid 2024-04-22T21:42:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-22T21:42:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: osdmap e157: 8 total, 8 up, 8 in 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:42:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:42:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:42:56.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: pgmap v250: 161 pgs: 19 active+undersized+degraded, 32 active+undersized, 110 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 62/627 objects degraded (9.888%) 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all osd 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: Upgrade: Setting require_osd_release to 19 squid 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: osdmap e157: 8 total, 8 up, 8 in 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:42:56.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:42:56.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:42:56.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:42:56.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:42:56.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T21:42:56.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:42:56.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:42:56.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:56.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:42:56.052 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.053 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:56.053 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:42:56.053 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:42:56.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:42:56.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:56.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:42:56.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:56.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:42:56.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:55 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: pgmap v250: 161 pgs: 19 active+undersized+degraded, 32 active+undersized, 110 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 62/627 objects degraded (9.888%) 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all osd 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: Upgrade: Setting require_osd_release to 19 squid 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: osdmap e157: 8 total, 8 up, 8 in 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:42:56.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:42:56.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:42:56.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:42:56.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:42:56.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:42:56.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:56.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:42:56.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:42:56.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:42:56.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:55 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps' 2024-04-22T21:42:56.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mds 2024-04-22T21:42:56.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:42:56.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:42:56.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all nfs 2024-04-22T21:42:56.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:42:56.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[82829]: Upgrade: Finalizing container_image settings 2024-04-22T21:42:56.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[82829]: Upgrade: Complete! 2024-04-22T21:42:56.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:56.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mds 2024-04-22T21:42:56.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:42:56.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:42:56.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all nfs 2024-04-22T21:42:56.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:42:56.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[85205]: Upgrade: Finalizing container_image settings 2024-04-22T21:42:56.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[85205]: Upgrade: Complete! 2024-04-22T21:42:56.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:56 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:57.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:56 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mds 2024-04-22T21:42:57.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:56 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:42:57.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:56 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:42:57.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:56 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all nfs 2024-04-22T21:42:57.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:56 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:42:57.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:56 smithi150 ceph-mon[65103]: Upgrade: Finalizing container_image settings 2024-04-22T21:42:57.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:56 smithi150 ceph-mon[65103]: Upgrade: Complete! 2024-04-22T21:42:57.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:56 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:42:57.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:57 smithi130 ceph-mon[82829]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:57.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:57 smithi130 ceph-mon[82829]: from='client.54606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:57.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:57 smithi130 ceph-mon[82829]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 62/627 objects degraded (9.888%), 19 pgs degraded) 2024-04-22T21:42:57.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:57 smithi130 ceph-mon[82829]: Cluster is now healthy 2024-04-22T21:42:57.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:57 smithi130 ceph-mon[85205]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:57.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:57 smithi130 ceph-mon[85205]: from='client.54606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:57.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:57 smithi130 ceph-mon[85205]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 62/627 objects degraded (9.888%), 19 pgs degraded) 2024-04-22T21:42:57.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:57 smithi130 ceph-mon[85205]: Cluster is now healthy 2024-04-22T21:42:58.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:57 smithi150 ceph-mon[65103]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:42:58.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:57 smithi150 ceph-mon[65103]: from='client.54606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:58.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:57 smithi150 ceph-mon[65103]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 62/627 objects degraded (9.888%), 19 pgs degraded) 2024-04-22T21:42:58.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:57 smithi150 ceph-mon[65103]: Cluster is now healthy 2024-04-22T21:42:58.517 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:42:58.517 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (9m) 2m ago 31m 27.7M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:42:58.517 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (8m) 9s ago 31m 83.0M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:42:58.517 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (8m) 2m ago 30m 60.5M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:42:58.517 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (10m) 9s ago 36m 454M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:42:58.517 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (20m) 2m ago 37m 510M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:42:58.517 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (7m) 2m ago 38m 55.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:42:58.517 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (7m) 9s ago 36m 50.3M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:42:58.517 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (6m) 2m ago 36m 41.2M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:42:58.517 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (10m) 2m ago 32m 15.7M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:42:58.518 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (10m) 9s ago 32m 15.8M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:42:58.518 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (4m) 2m ago 36m 47.6M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:42:58.518 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (4m) 2m ago 35m 52.4M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:42:58.518 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (3m) 2m ago 35m 41.5M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 027e1ac8c177 2024-04-22T21:42:58.518 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (2m) 2m ago 34m 12.1M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 ca6c16e44590 2024-04-22T21:42:58.518 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (118s) 9s ago 34m 55.6M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 0c80f1b448cb 2024-04-22T21:42:58.518 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (84s) 9s ago 34m 47.7M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 aef1942f3f6b 2024-04-22T21:42:58.518 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (48s) 9s ago 33m 52.2M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 d84338a30cd0 2024-04-22T21:42:58.518 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (14s) 9s ago 33m 12.6M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 d0d3f15f9e88 2024-04-22T21:42:58.518 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (9m) 9s ago 32m 72.8M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:42:58.518 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (30m) 2m ago 30m 80.2M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:42:58.518 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (30m) 9s ago 30m 78.6M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:42:59.268 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:42:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:42:58] "GET /metrics HTTP/1.1" 200 38165 "" "Prometheus/2.43.0" 2024-04-22T21:42:59.295 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-04-22T21:42:59.889 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:59 smithi130 ceph-mon[82829]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 684 B/s rd, 0 op/s 2024-04-22T21:42:59.889 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:59 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:59.889 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:42:59 smithi130 ceph-mon[82829]: from='client.54612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:42:59.889 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:59 smithi130 ceph-mon[85205]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 684 B/s rd, 0 op/s 2024-04-22T21:42:59.889 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:59 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:42:59.889 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:42:59 smithi130 ceph-mon[85205]: from='client.54612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:00.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:59 smithi150 ceph-mon[65103]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 684 B/s rd, 0 op/s 2024-04-22T21:43:00.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:59 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:00.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:42:59 smithi150 ceph-mon[65103]: from='client.54612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:01.541 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:43:01.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:01 smithi130 ceph-mon[82829]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:01.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:01 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:01.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:01 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/4211764335' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:01.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:01 smithi130 ceph-mon[85205]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:01.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:01 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:01.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:01 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/4211764335' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:02.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:01 smithi150 ceph-mon[65103]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:02.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:01 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:02.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:01 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/4211764335' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:02.318 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-04-22T21:43:03.895 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:03 smithi130 ceph-mon[82829]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:43:03.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:03 smithi130 ceph-mon[85205]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:43:04.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:03 smithi150 ceph-mon[65103]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:43:04.376 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)" 2024-04-22T21:43:04.997 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:04 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/3601451444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:04.997 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:04 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/3601451444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:05.020 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade status' 2024-04-22T21:43:05.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:04 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/3601451444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:06.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:05 smithi130 ceph-mon[82829]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:43:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:05 smithi130 ceph-mon[85205]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:43:06.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:05 smithi150 ceph-mon[65103]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:43:07.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:07.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:43:07.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:07.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:43:07.060 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:43:07.060 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": null, 2024-04-22T21:43:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": false, 2024-04-22T21:43:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "", 2024-04-22T21:43:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:43:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": null, 2024-04-22T21:43:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "", 2024-04-22T21:43:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:43:07.061 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:43:07.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:07.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:43:07.742 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph health detail' 2024-04-22T21:43:08.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:07 smithi130 ceph-mon[82829]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 940 B/s rd, 0 op/s 2024-04-22T21:43:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:07 smithi130 ceph-mon[85205]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 940 B/s rd, 0 op/s 2024-04-22T21:43:08.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:07 smithi150 ceph-mon[65103]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 940 B/s rd, 0 op/s 2024-04-22T21:43:09.034 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:08 smithi130 ceph-mon[85205]: from='client.64564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:09.035 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:43:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:43:08] "GET /metrics HTTP/1.1" 200 38311 "" "Prometheus/2.43.0" 2024-04-22T21:43:09.035 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:08 smithi130 ceph-mon[82829]: from='client.64564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:09.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:08 smithi150 ceph-mon[65103]: from='client.64564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:09.780 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-04-22T21:43:10.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:09 smithi130 ceph-mon[82829]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:10.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:09 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:09 smithi130 ceph-mon[85205]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:09 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:10.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:09 smithi150 ceph-mon[65103]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:10.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:09 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:10.509 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-04-22T21:43:10.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:10 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/3236369635' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:43:10.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:10 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/3236369635' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:43:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:10 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/3236369635' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:43:12.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:11 smithi130 ceph-mon[82829]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:12.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:11 smithi130 ceph-mon[85205]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:12.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:11 smithi150 ceph-mon[65103]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:13.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:13 smithi130 ceph-mon[82829]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:13.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:13 smithi130 ceph-mon[82829]: from='client.54642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:13.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:13 smithi130 ceph-mon[85205]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:13.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:13 smithi130 ceph-mon[85205]: from='client.54642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:14.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:13 smithi150 ceph-mon[65103]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:14.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:13 smithi150 ceph-mon[65103]: from='client.54642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:14.306 INFO:teuthology.orchestra.run.smithi130.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:43:15.021 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T21:43:15.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[85205]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:15.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[85205]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:43:15.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:15.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:15.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:43:15.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:15.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[85205]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:43:15.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[82829]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:15.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[82829]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:43:15.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:15.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:15.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:43:15.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:15.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:15 smithi130 ceph-mon[82829]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:43:15.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:15 smithi150 ceph-mon[65103]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:15.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:15 smithi150 ceph-mon[65103]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:43:15.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:15 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:15.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:15 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:15.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:15 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:43:15.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:15 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:15.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:15 smithi150 ceph-mon[65103]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:43:17.433 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:17.433 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.433 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:43:17.433 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:43:17.433 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:43:17.433 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.433 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mgr 2024-04-22T21:43:17.433 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.433 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.433 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mon 2024-04-22T21:43:17.433 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.433 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.434 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all crash 2024-04-22T21:43:17.434 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.434 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.434 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all osd 2024-04-22T21:43:17.434 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.434 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.434 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mds 2024-04-22T21:43:17.434 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:17.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:43:17.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mgr 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mon 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all crash 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all osd 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mds 2024-04-22T21:43:17.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:17 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:17.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:43:17.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:43:17.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:43:17.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mgr 2024-04-22T21:43:17.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mon 2024-04-22T21:43:17.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all crash 2024-04-22T21:43:17.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all osd 2024-04-22T21:43:17.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:17.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mds 2024-04-22T21:43:17.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:17 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:17.920 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:43:18.593 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:43:18.593 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (9m) 2m ago 31m 27.7M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:43:18.593 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (9m) 29s ago 31m 83.0M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (9m) 2m ago 31m 60.5M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (11m) 29s ago 36m 454M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (20m) 2m ago 38m 510M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (7m) 2m ago 38m 55.0M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (8m) 29s ago 36m 50.3M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (6m) 2m ago 37m 41.2M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (10m) 2m ago 32m 15.7M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (10m) 29s ago 32m 15.8M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (5m) 2m ago 36m 47.6M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (4m) 2m ago 35m 52.4M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (3m) 2m ago 35m 41.5M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 027e1ac8c177 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (2m) 2m ago 35m 12.1M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 ca6c16e44590 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (2m) 29s ago 34m 55.6M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 0c80f1b448cb 2024-04-22T21:43:18.594 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (104s) 29s ago 34m 47.7M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 aef1942f3f6b 2024-04-22T21:43:18.595 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (69s) 29s ago 33m 52.2M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 d84338a30cd0 2024-04-22T21:43:18.595 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (34s) 29s ago 33m 12.6M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 d0d3f15f9e88 2024-04-22T21:43:18.595 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (10m) 29s ago 32m 72.8M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:43:18.595 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (31m) 2m ago 31m 80.2M - 17.2.0 e1d6a67b021e 7a6164cee382 2024-04-22T21:43:18.595 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (31m) 29s ago 31m 78.6M - 17.2.0 e1d6a67b021e ba36b36ef4df 2024-04-22T21:43:19.022 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:43:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:43:18] "GET /metrics HTTP/1.1" 200 38311 "" "Prometheus/2.43.0" 2024-04-22T21:43:19.023 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:43:19.023 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:43:19.023 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:43:19.023 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:43:19.023 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:43:19.023 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:43:19.024 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:43:19.024 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:43:19.024 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 8 2024-04-22T21:43:19.024 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:43:19.024 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:43:19.024 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T21:43:19.024 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:43:19.025 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:43:19.025 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-22T21:43:19.025 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 13 2024-04-22T21:43:19.025 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:43:19.025 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:43:19.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[82829]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:19.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[82829]: from='client.64582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:19.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[82829]: from='client.54651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:19.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:19.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[82829]: from='client.64594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:19.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:19.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi130.dfmola", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:43:19.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:19.295 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/2063444589' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:19.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[85205]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:19.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[85205]: from='client.64582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:19.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[85205]: from='client.54651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:19.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:19.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[85205]: from='client.64594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:19.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:19.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi130.dfmola", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:43:19.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:19.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:19 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/2063444589' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:19.370 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:43:19.371 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:43:19.371 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:43:19.371 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-04-22T21:43:19.371 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:43:19.371 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "0/2 daemons upgraded", 2024-04-22T21:43:19.372 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading rgw daemons", 2024-04-22T21:43:19.372 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:43:19.372 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:43:19.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:19 smithi150 ceph-mon[65103]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:19.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:19 smithi150 ceph-mon[65103]: from='client.64582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:19.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:19 smithi150 ceph-mon[65103]: from='client.54651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:19.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:19 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:19.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:19 smithi150 ceph-mon[65103]: from='client.64594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:19.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:19 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:19.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:19 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi130.dfmola", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:43:19.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:19 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:19.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:19 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/2063444589' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:20.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:20 smithi130 ceph-mon[85205]: Upgrade: Updating rgw.foo.smithi130.dfmola (1/2) 2024-04-22T21:43:20.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:20 smithi130 ceph-mon[85205]: Deploying daemon rgw.foo.smithi130.dfmola on smithi130 2024-04-22T21:43:20.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:20 smithi130 ceph-mon[85205]: from='client.54669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:20.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:20 smithi130 ceph-mon[82829]: Upgrade: Updating rgw.foo.smithi130.dfmola (1/2) 2024-04-22T21:43:20.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:20 smithi130 ceph-mon[82829]: Deploying daemon rgw.foo.smithi130.dfmola on smithi130 2024-04-22T21:43:20.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:20 smithi130 ceph-mon[82829]: from='client.54669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:20.547 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:20 smithi150 ceph-mon[65103]: Upgrade: Updating rgw.foo.smithi130.dfmola (1/2) 2024-04-22T21:43:20.547 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:20 smithi150 ceph-mon[65103]: Deploying daemon rgw.foo.smithi130.dfmola on smithi130 2024-04-22T21:43:20.547 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:20 smithi150 ceph-mon[65103]: from='client.54669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:21.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:21 smithi130 ceph-mon[82829]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:21.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:43:21.508 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:21 smithi130 ceph-mon[85205]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:21.508 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:43:21.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:21 smithi150 ceph-mon[65103]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:21.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:43:23.510 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:23 smithi150 ceph-mon[65103]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:23.511 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:23 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:23.511 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:23 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:23.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:23 smithi130 ceph-mon[85205]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:23.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:23 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:23.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:23 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:23.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:23 smithi130 ceph-mon[82829]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:23.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:23 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:23.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:23 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:25.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:25 smithi130 ceph-mon[82829]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:43:25.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:25 smithi130 ceph-mon[82829]: Upgrade: Updating rgw.foo.smithi150.xqvixd (2/2) 2024-04-22T21:43:25.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:25 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:25.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:25 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi150.xqvixd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:43:25.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:25 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:25.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:25 smithi130 ceph-mon[82829]: Deploying daemon rgw.foo.smithi150.xqvixd on smithi150 2024-04-22T21:43:25.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:25 smithi130 ceph-mon[85205]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:43:25.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:25 smithi130 ceph-mon[85205]: Upgrade: Updating rgw.foo.smithi150.xqvixd (2/2) 2024-04-22T21:43:25.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:25 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:25.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:25 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi150.xqvixd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:43:25.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:25 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:25.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:25 smithi130 ceph-mon[85205]: Deploying daemon rgw.foo.smithi150.xqvixd on smithi150 2024-04-22T21:43:25.812 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:25 smithi150 ceph-mon[65103]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T21:43:25.812 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:25 smithi150 ceph-mon[65103]: Upgrade: Updating rgw.foo.smithi150.xqvixd (2/2) 2024-04-22T21:43:25.812 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:25 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:25.812 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:25 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi150.xqvixd", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T21:43:25.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:25 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:25.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:25 smithi150 ceph-mon[65103]: Deploying daemon rgw.foo.smithi150.xqvixd on smithi150 2024-04-22T21:43:27.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:27 smithi130 ceph-mon[85205]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 59 op/s 2024-04-22T21:43:27.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:27 smithi130 ceph-mon[82829]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 59 op/s 2024-04-22T21:43:27.824 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:27 smithi150 ceph-mon[65103]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 59 op/s 2024-04-22T21:43:29.292 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:43:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:43:28] "GET /metrics HTTP/1.1" 200 38310 "" "Prometheus/2.43.0" 2024-04-22T21:43:29.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:29 smithi130 ceph-mon[85205]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-04-22T21:43:29.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:29 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:29.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:29 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:29.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:29 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:29.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:29 smithi130 ceph-mon[82829]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-04-22T21:43:29.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:29 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:29.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:29 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:29.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:29 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:29.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:29 smithi150 ceph-mon[65103]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-04-22T21:43:29.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:29 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:29.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:29 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:29.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:29 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:31.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:31 smithi130 ceph-mon[85205]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 85 B/s wr, 135 op/s 2024-04-22T21:43:31.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:31 smithi130 ceph-mon[82829]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 85 B/s wr, 135 op/s 2024-04-22T21:43:31.829 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:31 smithi150 ceph-mon[65103]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 85 B/s wr, 135 op/s 2024-04-22T21:43:33.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:33 smithi130 ceph-mon[85205]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 170 B/s wr, 148 op/s 2024-04-22T21:43:33.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:33 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:33.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:33 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:33.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:33 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:33.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:33 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:33.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:33 smithi130 ceph-mon[82829]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 170 B/s wr, 148 op/s 2024-04-22T21:43:33.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:33 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:33.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:33 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:33.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:33 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:33.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:33 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:33.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:33 smithi150 ceph-mon[65103]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 170 B/s wr, 148 op/s 2024-04-22T21:43:33.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:33 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:33.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:33 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:33.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:33 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:33.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:33 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:34.266 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:34 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:34.266 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:34 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:34.266 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:34 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:34.266 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:34 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:34.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:34 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:34.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:34 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:34.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:34 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:34.545 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:34 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:34.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:34 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:34.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:34 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:34.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:34 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:34.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:34 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:35.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:35 smithi150 ceph-mon[65103]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-04-22T21:43:35.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:35 smithi130 ceph-mon[85205]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-04-22T21:43:35.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:35 smithi130 ceph-mon[82829]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-04-22T21:43:36.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:43:36.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:43:36.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:43:38.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 170 B/s wr, 157 op/s 2024-04-22T21:43:38.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:38.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:43:38.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:43:38.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi130.dfmola"}]: dispatch 2024-04-22T21:43:38.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi130.dfmola"}]': finished 2024-04-22T21:43:38.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi150.xqvixd"}]: dispatch 2024-04-22T21:43:38.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi150.xqvixd"}]': finished 2024-04-22T21:43:38.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 170 B/s wr, 157 op/s 2024-04-22T21:43:38.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:38.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:43:38.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:43:38.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi130.dfmola"}]: dispatch 2024-04-22T21:43:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi130.dfmola"}]': finished 2024-04-22T21:43:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi150.xqvixd"}]: dispatch 2024-04-22T21:43:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi150.xqvixd"}]': finished 2024-04-22T21:43:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.053 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.053 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.053 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.053 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.053 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.053 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.053 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:43:38.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:43:38.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:43:38.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:43:38.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:43:38.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:43:38.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T21:43:38.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:43:38.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:43:38.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:43:38.054 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-22T21:43:38.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:43:38.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:43:38.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:43:38.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:43:38.055 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.055 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.055 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.055 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.055 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:43:38.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:43:38.057 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.057 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:43:38.057 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:43:38.057 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:43:38.057 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:43:38.057 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:43:38.057 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:43:38.057 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:43:38.057 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.057 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.057 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:43:38.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:43:38.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:38.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:43:38.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:38.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:43:38.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:43:38.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:38.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:43:38.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:37 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 170 B/s wr, 157 op/s 2024-04-22T21:43:38.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:38.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:43:38.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:43:38.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi130.dfmola"}]: dispatch 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi130.dfmola"}]': finished 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi150.xqvixd"}]: dispatch 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi150.xqvixd"}]': finished 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T21:43:38.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T21:43:38.102 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T21:43:38.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T21:43:38.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T21:43:38.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.103 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T21:43:38.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T21:43:38.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T21:43:38.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:38.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:43:38.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:43:38.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:43:38.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:37 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:38.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all rgw 2024-04-22T21:43:38.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:43:38.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:43:38.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all nfs 2024-04-22T21:43:38.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:43:38.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[82829]: Upgrade: Finalizing container_image settings 2024-04-22T21:43:38.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[82829]: Upgrade: Complete! 2024-04-22T21:43:38.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[82829]: Checking dashboard <-> RGW credentials 2024-04-22T21:43:38.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all rgw 2024-04-22T21:43:38.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:43:38.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:43:38.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all nfs 2024-04-22T21:43:38.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:43:38.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[85205]: Upgrade: Finalizing container_image settings 2024-04-22T21:43:38.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[85205]: Upgrade: Complete! 2024-04-22T21:43:38.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-mon[85205]: Checking dashboard <-> RGW credentials 2024-04-22T21:43:39.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:38 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all rgw 2024-04-22T21:43:39.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:38 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:43:39.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:38 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:43:39.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:38 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all nfs 2024-04-22T21:43:39.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:38 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all nvmeof 2024-04-22T21:43:39.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:38 smithi150 ceph-mon[65103]: Upgrade: Finalizing container_image settings 2024-04-22T21:43:39.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:38 smithi150 ceph-mon[65103]: Upgrade: Complete! 2024-04-22T21:43:39.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:38 smithi150 ceph-mon[65103]: Checking dashboard <-> RGW credentials 2024-04-22T21:43:39.294 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:43:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:43:38] "GET /metrics HTTP/1.1" 200 38344 "" "Prometheus/2.43.0" 2024-04-22T21:43:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:39 smithi130 ceph-mon[85205]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 170 B/s wr, 109 op/s 2024-04-22T21:43:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:39 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:40.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:39 smithi130 ceph-mon[82829]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 170 B/s wr, 109 op/s 2024-04-22T21:43:40.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:39 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:40.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:39 smithi150 ceph-mon[65103]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 170 B/s wr, 109 op/s 2024-04-22T21:43:40.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:39 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:41 smithi130 ceph-mon[85205]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 85 B/s wr, 85 op/s 2024-04-22T21:43:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:41 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:42.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:41 smithi130 ceph-mon[82829]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 85 B/s wr, 85 op/s 2024-04-22T21:43:42.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:41 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:42.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:41 smithi150 ceph-mon[65103]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 85 B/s wr, 85 op/s 2024-04-22T21:43:42.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:41 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:43:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:43 smithi130 ceph-mon[85205]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 85 B/s wr, 35 op/s 2024-04-22T21:43:44.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:43 smithi130 ceph-mon[82829]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 85 B/s wr, 35 op/s 2024-04-22T21:43:44.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:43 smithi150 ceph-mon[65103]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 85 B/s wr, 35 op/s 2024-04-22T21:43:46.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:45 smithi130 ceph-mon[85205]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 22 op/s 2024-04-22T21:43:46.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:45 smithi130 ceph-mon[82829]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 22 op/s 2024-04-22T21:43:46.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:45 smithi150 ceph-mon[65103]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 22 op/s 2024-04-22T21:43:47.880 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:47 smithi130 ceph-mon[85205]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-22T21:43:47.881 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:47 smithi130 ceph-mon[82829]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-22T21:43:48.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:47 smithi150 ceph-mon[65103]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-22T21:43:49.294 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:43:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:43:48] "GET /metrics HTTP/1.1" 200 38344 "" "Prometheus/2.43.0" 2024-04-22T21:43:49.900 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:49 smithi150 ceph-mon[65103]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-22T21:43:49.900 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:49 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:50.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:49 smithi130 ceph-mon[82829]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-22T21:43:50.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:49 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:49 smithi130 ceph-mon[85205]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-22T21:43:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:49 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:43:50.572 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps' 2024-04-22T21:43:50.897 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:50 smithi130 ceph-mon[82829]: from='client.64690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:50.897 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:50 smithi130 ceph-mon[85205]: from='client.64690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:51.063 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:50 smithi150 ceph-mon[65103]: from='client.64690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:51.993 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:51 smithi130 ceph-mon[82829]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-22T21:43:51.993 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:43:51.993 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:51 smithi130 ceph-mon[85205]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-22T21:43:51.993 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:43:52.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:51 smithi150 ceph-mon[65103]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-22T21:43:52.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:43:52.553 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:43:52.553 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (10m) 20s ago 32m 27.3M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:43:52.553 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (9m) 20s ago 32m 82.9M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:43:52.553 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (9m) 20s ago 31m 62.9M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:43:52.553 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (11m) 20s ago 37m 454M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:43:52.553 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (21m) 20s ago 38m 515M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:43:52.553 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (7m) 20s ago 38m 64.4M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:43:52.553 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (8m) 20s ago 37m 52.8M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:43:52.554 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (7m) 20s ago 37m 48.1M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:43:52.554 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (11m) 20s ago 33m 16.3M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:43:52.554 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (11m) 20s ago 33m 15.1M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:43:52.554 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (5m) 20s ago 36m 56.7M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:43:52.554 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (5m) 20s ago 36m 61.5M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:43:52.554 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (4m) 20s ago 36m 47.6M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 027e1ac8c177 2024-04-22T21:43:52.554 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (3m) 20s ago 35m 59.0M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 ca6c16e44590 2024-04-22T21:43:52.554 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (2m) 20s ago 35m 58.9M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 0c80f1b448cb 2024-04-22T21:43:52.554 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (2m) 20s ago 34m 50.9M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 aef1942f3f6b 2024-04-22T21:43:52.554 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (103s) 20s ago 34m 54.8M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 d84338a30cd0 2024-04-22T21:43:52.554 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (68s) 20s ago 34m 55.0M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 d0d3f15f9e88 2024-04-22T21:43:52.554 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (10m) 20s ago 33m 72.8M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:43:52.554 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (30s) 20s ago 31m 75.2M - 19.0.0-3198-g56e81df3 82086b19bb07 f0fe6481306a 2024-04-22T21:43:52.555 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (24s) 20s ago 31m 75.0M - 19.0.0-3198-g56e81df3 82086b19bb07 b226f7943446 2024-04-22T21:43:53.228 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-04-22T21:43:53.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:53 smithi130 ceph-mon[85205]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-22T21:43:53.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:53 smithi130 ceph-mon[85205]: from='client.54759 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:53.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:53 smithi130 ceph-mon[82829]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-22T21:43:53.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:53 smithi130 ceph-mon[82829]: from='client.54759 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:54.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:53 smithi150 ceph-mon[65103]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-22T21:43:54.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:53 smithi150 ceph-mon[65103]: from='client.54759 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:43:55.172 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:43:55.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:55 smithi130 ceph-mon[82829]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:55.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:55 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/1389348096' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:55.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:55 smithi130 ceph-mon[85205]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:55.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:55 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/1389348096' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:55.843 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-04-22T21:43:56.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:55 smithi150 ceph-mon[65103]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:43:56.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:55 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/1389348096' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:57.978 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)" 2024-04-22T21:43:57.978 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:57 smithi130 ceph-mon[85205]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:57.979 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:57 smithi130 ceph-mon[82829]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:58.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:57 smithi150 ceph-mon[65103]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:43:58.779 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade status' 2024-04-22T21:43:59.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:58 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/2881496159' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:59.044 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:43:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:43:58] "GET /metrics HTTP/1.1" 200 38344 "" "Prometheus/2.43.0" 2024-04-22T21:43:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:58 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/2881496159' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:43:59.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:58 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/2881496159' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:00.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:59 smithi130 ceph-mon[82829]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:00.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:43:59 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:59 smithi130 ceph-mon[85205]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:00.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:43:59 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:00.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:59 smithi150 ceph-mon[65103]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:00.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:43:59 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:00.812 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:44:00.812 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": null, 2024-04-22T21:44:00.812 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": false, 2024-04-22T21:44:00.812 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "", 2024-04-22T21:44:00.812 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-04-22T21:44:00.812 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": null, 2024-04-22T21:44:00.812 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "", 2024-04-22T21:44:00.812 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:44:00.812 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:44:01.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph health detail' 2024-04-22T21:44:01.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:01 smithi130 ceph-mon[82829]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:01.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:01 smithi130 ceph-mon[85205]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:02.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:01 smithi150 ceph-mon[65103]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:02.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:02 smithi130 ceph-mon[82829]: from='client.64714 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:02.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:02 smithi130 ceph-mon[85205]: from='client.64714 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:03.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:02 smithi150 ceph-mon[65103]: from='client.64714 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:03.597 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:44:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=cleanup t=2024-04-22T21:44:03.305773864Z level=info msg="Completed cleanup jobs" duration=3.731256ms 2024-04-22T21:44:03.631 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-04-22T21:44:04.014 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:03 smithi130 ceph-mon[85205]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:04.014 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:03 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/2141657932' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:44:04.014 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:03 smithi130 ceph-mon[82829]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:04.014 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:03 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/2141657932' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:44:04.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:03 smithi150 ceph-mon[65103]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:04.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:03 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/2141657932' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:44:04.410 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-22T21:44:05.948 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:05 smithi130 ceph-mon[82829]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:05.949 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:05 smithi130 ceph-mon[85205]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:06.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:05 smithi150 ceph-mon[65103]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:06.396 INFO:teuthology.orchestra.run.smithi130.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:44:07.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:44:07.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:07.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:44:07.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:44:07.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:07.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:44:07.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:07.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:44:07.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:44:07.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:07.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:44:07.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:07.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:44:07.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:44:07.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:07.191 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T21:44:07.205 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi130.front.sepia.ceph.com 2024-04-22T21:44:07.205 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-22T21:44:07.775 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:07 smithi130 ceph-mon[82829]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:07.775 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:07 smithi130 ceph-mon[82829]: from='client.64726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:07.775 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:07 smithi130 ceph-mon[82829]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:44:07.775 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:07 smithi130 ceph-mon[82829]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:44:07.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:07 smithi130 ceph-mon[85205]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:07.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:07 smithi130 ceph-mon[85205]: from='client.64726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:07.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:07 smithi130 ceph-mon[85205]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:44:07.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:07 smithi130 ceph-mon[85205]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:44:08.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:07 smithi150 ceph-mon[65103]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:08.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:07 smithi150 ceph-mon[65103]: from='client.64726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:08.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:07 smithi150 ceph-mon[65103]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:44:08.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:07 smithi150 ceph-mon[65103]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:44:09.079 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:44:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:44:08] "GET /metrics HTTP/1.1" 200 38345 "" "Prometheus/2.43.0" 2024-04-22T21:44:09.944 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:09.944 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:09.944 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.944 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:44:09.944 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.946 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.946 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.946 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:09.946 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:09.946 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.946 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:44:09.946 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.946 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.946 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.946 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.946 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.947 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.947 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.947 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.947 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.947 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.947 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.947 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.947 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.947 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:09.947 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.947 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:09.947 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:09 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:10.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:10.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:10.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:10.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:10.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:10.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:10.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:09 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:11.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[82829]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:44:11.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[82829]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:44:11.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mgr 2024-04-22T21:44:11.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mon 2024-04-22T21:44:11.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all crash 2024-04-22T21:44:11.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all osd 2024-04-22T21:44:11.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all mds 2024-04-22T21:44:11.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all rgw 2024-04-22T21:44:11.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:44:11.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T21:44:11.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[82829]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:44:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[85205]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:44:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[85205]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:44:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mgr 2024-04-22T21:44:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mon 2024-04-22T21:44:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all crash 2024-04-22T21:44:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all osd 2024-04-22T21:44:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all mds 2024-04-22T21:44:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all rgw 2024-04-22T21:44:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:44:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T21:44:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:10 smithi130 ceph-mon[85205]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:44:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:10 smithi150 ceph-mon[65103]: Upgrade: Target is version 19.0.0-3198-g56e81df3 (squid) 2024-04-22T21:44:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:10 smithi150 ceph-mon[65103]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90c23f6ff09279e806b61a7a0a9b6863a8d239088a6c790da6364c3d6ab9ae8d'] 2024-04-22T21:44:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:10 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mgr 2024-04-22T21:44:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:10 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mon 2024-04-22T21:44:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:10 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all crash 2024-04-22T21:44:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:10 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all osd 2024-04-22T21:44:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:10 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all mds 2024-04-22T21:44:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:10 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all rgw 2024-04-22T21:44:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:10 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T21:44:11.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:10 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T21:44:11.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:10 smithi150 ceph-mon[65103]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T21:44:11.599 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:44:11.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:11 smithi130 ceph-mon[85205]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:11.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:11 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:11.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:11 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:44:11.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:11 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:44:11.861 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:11 smithi130 ceph-mon[82829]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:11.861 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:11 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:11.861 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:11 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:44:11.862 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:11 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:44:12.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:11 smithi150 ceph-mon[65103]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:12.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:11 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:12.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:11 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi130.pcsiph", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T21:44:12.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:11 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:44:12.271 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (10m) 40s ago 32m 27.3M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (10m) 39s ago 32m 82.9M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (9m) 40s ago 31m 62.9M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (11m) 39s ago 37m 454M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (21m) 40s ago 39m 515M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (8m) 40s ago 39m 64.4M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (9m) 39s ago 37m 52.8M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (7m) 40s ago 38m 48.1M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (11m) 40s ago 33m 16.3M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (11m) 39s ago 33m 15.1M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (6m) 40s ago 37m 56.7M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (5m) 40s ago 36m 61.5M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (4m) 40s ago 36m 47.6M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 027e1ac8c177 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (3m) 40s ago 36m 59.0M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 ca6c16e44590 2024-04-22T21:44:12.272 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (3m) 39s ago 35m 58.9M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 0c80f1b448cb 2024-04-22T21:44:12.273 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (2m) 39s ago 35m 50.9M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 aef1942f3f6b 2024-04-22T21:44:12.273 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (2m) 39s ago 34m 54.8M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 d84338a30cd0 2024-04-22T21:44:12.273 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (88s) 39s ago 34m 55.0M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 d0d3f15f9e88 2024-04-22T21:44:12.273 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (11m) 39s ago 33m 72.8M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:44:12.273 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (49s) 40s ago 32m 75.2M - 19.0.0-3198-g56e81df3 82086b19bb07 f0fe6481306a 2024-04-22T21:44:12.273 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (44s) 39s ago 32m 75.0M - 19.0.0-3198-g56e81df3 82086b19bb07 b226f7943446 2024-04-22T21:44:12.673 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:44:12.673 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:44:12.673 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:44:12.673 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:44:12.673 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:44:12.673 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:44:12.673 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:44:12.673 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:44:12.673 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 8 2024-04-22T21:44:12.673 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:44:12.673 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:44:12.673 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:44:12.674 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:44:12.674 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:44:12.674 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 15 2024-04-22T21:44:12.674 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:44:12.674 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:44:12.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:12 smithi130 ceph-mon[85205]: Upgrade: Updating iscsi.foo.smithi130.pcsiph 2024-04-22T21:44:12.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:12 smithi130 ceph-mon[85205]: Deploying daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:44:12.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:12 smithi130 ceph-mon[85205]: from='client.54795 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:12.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:12 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/3214045834' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:12.961 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:12 smithi130 ceph-mon[82829]: Upgrade: Updating iscsi.foo.smithi130.pcsiph 2024-04-22T21:44:12.961 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:12 smithi130 ceph-mon[82829]: Deploying daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:44:12.961 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:12 smithi130 ceph-mon[82829]: from='client.54795 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:12.961 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:12 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/3214045834' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:13.015 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:44:13.015 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:44:13.015 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:44:13.015 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T21:44:13.015 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-04-22T21:44:13.016 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr", 2024-04-22T21:44:13.016 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw", 2024-04-22T21:44:13.016 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-04-22T21:44:13.016 INFO:teuthology.orchestra.run.smithi130.stdout: "osd" 2024-04-22T21:44:13.016 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-04-22T21:44:13.016 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "15/21 daemons upgraded", 2024-04-22T21:44:13.016 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading iscsi daemons", 2024-04-22T21:44:13.016 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-04-22T21:44:13.016 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:44:13.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:12 smithi150 ceph-mon[65103]: Upgrade: Updating iscsi.foo.smithi130.pcsiph 2024-04-22T21:44:13.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:12 smithi150 ceph-mon[65103]: Deploying daemon iscsi.foo.smithi130.pcsiph on smithi130 2024-04-22T21:44:13.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:12 smithi150 ceph-mon[65103]: from='client.54795 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:13.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:12 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/3214045834' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:13.440 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-04-22T21:44:14.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:13 smithi130 ceph-mon[82829]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:14.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:13 smithi130 ceph-mon[82829]: from='client.54801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:14.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:13 smithi130 ceph-mon[82829]: from='client.64744 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:14.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:13 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/2915598583' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:44:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:13 smithi130 ceph-mon[85205]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:13 smithi130 ceph-mon[85205]: from='client.54801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:13 smithi130 ceph-mon[85205]: from='client.64744 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:13 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/2915598583' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:44:14.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:13 smithi150 ceph-mon[65103]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:14.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:13 smithi150 ceph-mon[65103]: from='client.54801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:14.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:13 smithi150 ceph-mon[65103]: from='client.64744 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:14.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:13 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/2915598583' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:44:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:14 smithi130 ceph-mon[85205]: from='client.64756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:15.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:14 smithi130 ceph-mon[82829]: from='client.64756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:15.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:14 smithi150 ceph-mon[65103]: from='client.64756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:15 smithi130 ceph-mon[85205]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:16.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:15 smithi130 ceph-mon[82829]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:16.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:15 smithi150 ceph-mon[65103]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:18.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:17 smithi130 ceph-mon[82829]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:17 smithi130 ceph-mon[85205]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:18.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:17 smithi150 ceph-mon[65103]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:19.294 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:44:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:44:18] "GET /metrics HTTP/1.1" 200 38345 "" "Prometheus/2.43.0" 2024-04-22T21:44:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:19 smithi130 ceph-mon[85205]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:19 smithi130 ceph-mon[85205]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:20.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:19 smithi130 ceph-mon[82829]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:20.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:19 smithi130 ceph-mon[82829]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:20.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:19 smithi150 ceph-mon[65103]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:20.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:19 smithi150 ceph-mon[65103]: from='client.25714 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:21.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:20 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:44:21.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:20 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:44:21.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:20 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:44:22.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:21 smithi130 ceph-mon[82829]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:21 smithi130 ceph-mon[85205]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:22.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:21 smithi150 ceph-mon[65103]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:23.925 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:23 smithi130 ceph-mon[85205]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:23.925 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:23 smithi130 ceph-mon[82829]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:24.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:23 smithi150 ceph-mon[65103]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:25.293 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:44:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:44:25.008+0000 7fd6ea6b3700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi130.pcsiph on host smithi130 failed. 2024-04-22T21:44:25.979 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:25 smithi130 ceph-mon[82829]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:25.979 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:25 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:25.979 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:25 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:44:25.979 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:25 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:44:25.979 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:25 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:25.979 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:25 smithi130 ceph-mon[85205]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:25.979 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:25 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:25.979 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:25 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:44:25.979 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:25 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:44:25.980 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:25 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:26.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:25 smithi150 ceph-mon[65103]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:26.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:25 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:26.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:25 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:44:26.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:25 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:44:26.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:25 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:27.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:26 smithi130 ceph-mon[85205]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi130.pcsiph on host smithi130 failed. 2024-04-22T21:44:27.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:26 smithi130 ceph-mon[85205]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:27.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:26 smithi130 ceph-mon[85205]: Health check failed: Upgrading daemon iscsi.foo.smithi130.pcsiph on host smithi130 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-22T21:44:27.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:26 smithi130 ceph-mon[82829]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi130.pcsiph on host smithi130 failed. 2024-04-22T21:44:27.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:26 smithi130 ceph-mon[82829]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:27.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:26 smithi130 ceph-mon[82829]: Health check failed: Upgrading daemon iscsi.foo.smithi130.pcsiph on host smithi130 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-22T21:44:27.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:26 smithi150 ceph-mon[65103]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi130.pcsiph on host smithi130 failed. 2024-04-22T21:44:27.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:26 smithi150 ceph-mon[65103]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:27.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:26 smithi150 ceph-mon[65103]: Health check failed: Upgrading daemon iscsi.foo.smithi130.pcsiph on host smithi130 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-22T21:44:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:28 smithi130 ceph-mon[85205]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 820 B/s rd, 0 op/s 2024-04-22T21:44:29.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:28 smithi130 ceph-mon[82829]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 820 B/s rd, 0 op/s 2024-04-22T21:44:29.044 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:44:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:44:28] "GET /metrics HTTP/1.1" 200 38343 "" "Prometheus/2.43.0" 2024-04-22T21:44:29.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:28 smithi150 ceph-mon[65103]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 820 B/s rd, 0 op/s 2024-04-22T21:44:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:30 smithi130 ceph-mon[85205]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 820 B/s rd, 0 op/s 2024-04-22T21:44:31.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:30 smithi130 ceph-mon[82829]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 820 B/s rd, 0 op/s 2024-04-22T21:44:31.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:30 smithi150 ceph-mon[65103]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 820 B/s rd, 0 op/s 2024-04-22T21:44:33.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:32 smithi130 ceph-mon[85205]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 364 B/s rd, 0 op/s 2024-04-22T21:44:33.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:32 smithi130 ceph-mon[82829]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 364 B/s rd, 0 op/s 2024-04-22T21:44:33.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:32 smithi150 ceph-mon[65103]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 364 B/s rd, 0 op/s 2024-04-22T21:44:35.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:34 smithi130 ceph-mon[82829]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 364 B/s rd, 0 op/s 2024-04-22T21:44:35.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:34 smithi130 ceph-mon[85205]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 364 B/s rd, 0 op/s 2024-04-22T21:44:35.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:34 smithi150 ceph-mon[65103]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 364 B/s rd, 0 op/s 2024-04-22T21:44:37.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:36 smithi130 ceph-mon[82829]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 364 B/s rd, 0 op/s 2024-04-22T21:44:37.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:37.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:44:37.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:36 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/3561802008' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:44:37.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:36 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/202552036' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2123624486"}]: dispatch 2024-04-22T21:44:37.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:36 smithi130 ceph-mon[82829]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2123624486"}]: dispatch 2024-04-22T21:44:37.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:36 smithi130 ceph-mon[85205]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 364 B/s rd, 0 op/s 2024-04-22T21:44:37.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:37.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:44:37.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:36 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/3561802008' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:44:37.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:36 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/202552036' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2123624486"}]: dispatch 2024-04-22T21:44:37.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:36 smithi130 ceph-mon[85205]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2123624486"}]: dispatch 2024-04-22T21:44:37.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:36 smithi150 ceph-mon[65103]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 364 B/s rd, 0 op/s 2024-04-22T21:44:37.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:44:37.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:44:37.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:36 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/3561802008' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T21:44:37.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:36 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/202552036' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2123624486"}]: dispatch 2024-04-22T21:44:37.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:36 smithi150 ceph-mon[65103]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2123624486"}]: dispatch 2024-04-22T21:44:38.043 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:37 smithi130 ceph-mon[82829]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2123624486"}]': finished 2024-04-22T21:44:38.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:37 smithi130 ceph-mon[82829]: osdmap e158: 8 total, 8 up, 8 in 2024-04-22T21:44:38.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:37 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/2307991158' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/3974784268"}]: dispatch 2024-04-22T21:44:38.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:37 smithi130 ceph-mon[82829]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/3974784268"}]: dispatch 2024-04-22T21:44:38.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:37 smithi130 ceph-mon[85205]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2123624486"}]': finished 2024-04-22T21:44:38.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:37 smithi130 ceph-mon[85205]: osdmap e158: 8 total, 8 up, 8 in 2024-04-22T21:44:38.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:37 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/2307991158' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/3974784268"}]: dispatch 2024-04-22T21:44:38.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:37 smithi130 ceph-mon[85205]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/3974784268"}]: dispatch 2024-04-22T21:44:38.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:37 smithi150 ceph-mon[65103]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2123624486"}]': finished 2024-04-22T21:44:38.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:37 smithi150 ceph-mon[65103]: osdmap e158: 8 total, 8 up, 8 in 2024-04-22T21:44:38.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:37 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/2307991158' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/3974784268"}]: dispatch 2024-04-22T21:44:38.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:37 smithi150 ceph-mon[65103]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/3974784268"}]: dispatch 2024-04-22T21:44:39.044 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:44:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:44:38] "GET /metrics HTTP/1.1" 200 38425 "" "Prometheus/2.43.0" 2024-04-22T21:44:39.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:38 smithi130 ceph-mon[82829]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:44:39.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:38 smithi130 ceph-mon[82829]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/3974784268"}]': finished 2024-04-22T21:44:39.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:38 smithi130 ceph-mon[82829]: osdmap e159: 8 total, 8 up, 8 in 2024-04-22T21:44:39.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:38 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/25955405' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3411933457"}]: dispatch 2024-04-22T21:44:39.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:38 smithi130 ceph-mon[82829]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3411933457"}]: dispatch 2024-04-22T21:44:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:38 smithi130 ceph-mon[85205]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:44:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:38 smithi130 ceph-mon[85205]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/3974784268"}]': finished 2024-04-22T21:44:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:38 smithi130 ceph-mon[85205]: osdmap e159: 8 total, 8 up, 8 in 2024-04-22T21:44:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:38 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/25955405' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3411933457"}]: dispatch 2024-04-22T21:44:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:38 smithi130 ceph-mon[85205]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3411933457"}]: dispatch 2024-04-22T21:44:39.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:38 smithi150 ceph-mon[65103]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-04-22T21:44:39.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:38 smithi150 ceph-mon[65103]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6800/3974784268"}]': finished 2024-04-22T21:44:39.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:38 smithi150 ceph-mon[65103]: osdmap e159: 8 total, 8 up, 8 in 2024-04-22T21:44:39.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:38 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/25955405' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3411933457"}]: dispatch 2024-04-22T21:44:39.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:38 smithi150 ceph-mon[65103]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3411933457"}]: dispatch 2024-04-22T21:44:40.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:39 smithi130 ceph-mon[85205]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3411933457"}]': finished 2024-04-22T21:44:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:39 smithi130 ceph-mon[85205]: osdmap e160: 8 total, 8 up, 8 in 2024-04-22T21:44:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:39 smithi130 ceph-mon[85205]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:44:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:39 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/3229914763' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2248332625"}]: dispatch 2024-04-22T21:44:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:39 smithi130 ceph-mon[85205]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2248332625"}]: dispatch 2024-04-22T21:44:40.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:39 smithi130 ceph-mon[82829]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3411933457"}]': finished 2024-04-22T21:44:40.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:39 smithi130 ceph-mon[82829]: osdmap e160: 8 total, 8 up, 8 in 2024-04-22T21:44:40.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:39 smithi130 ceph-mon[82829]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:44:40.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:39 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/3229914763' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2248332625"}]: dispatch 2024-04-22T21:44:40.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:39 smithi130 ceph-mon[82829]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2248332625"}]: dispatch 2024-04-22T21:44:40.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:39 smithi150 ceph-mon[65103]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3411933457"}]': finished 2024-04-22T21:44:40.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:39 smithi150 ceph-mon[65103]: osdmap e160: 8 total, 8 up, 8 in 2024-04-22T21:44:40.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:39 smithi150 ceph-mon[65103]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T21:44:40.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:39 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/3229914763' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2248332625"}]: dispatch 2024-04-22T21:44:40.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:39 smithi150 ceph-mon[65103]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2248332625"}]: dispatch 2024-04-22T21:44:41.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:40 smithi130 ceph-mon[85205]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2248332625"}]': finished 2024-04-22T21:44:41.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:40 smithi130 ceph-mon[85205]: osdmap e161: 8 total, 8 up, 8 in 2024-04-22T21:44:41.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:40 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/272216143' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/3974784268"}]: dispatch 2024-04-22T21:44:41.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:40 smithi130 ceph-mon[85205]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/3974784268"}]: dispatch 2024-04-22T21:44:41.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:40 smithi130 ceph-mon[82829]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2248332625"}]': finished 2024-04-22T21:44:41.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:40 smithi130 ceph-mon[82829]: osdmap e161: 8 total, 8 up, 8 in 2024-04-22T21:44:41.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:40 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/272216143' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/3974784268"}]: dispatch 2024-04-22T21:44:41.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:40 smithi130 ceph-mon[82829]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/3974784268"}]: dispatch 2024-04-22T21:44:41.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:40 smithi150 ceph-mon[65103]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/2248332625"}]': finished 2024-04-22T21:44:41.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:40 smithi150 ceph-mon[65103]: osdmap e161: 8 total, 8 up, 8 in 2024-04-22T21:44:41.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:40 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/272216143' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/3974784268"}]: dispatch 2024-04-22T21:44:41.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:40 smithi150 ceph-mon[65103]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/3974784268"}]: dispatch 2024-04-22T21:44:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:41 smithi130 ceph-mon[85205]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/3974784268"}]': finished 2024-04-22T21:44:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:41 smithi130 ceph-mon[85205]: osdmap e162: 8 total, 8 up, 8 in 2024-04-22T21:44:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:41 smithi130 ceph-mon[85205]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:44:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:41 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/3747709372' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3568156832"}]: dispatch 2024-04-22T21:44:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:41 smithi130 ceph-mon[85205]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3568156832"}]: dispatch 2024-04-22T21:44:42.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:41 smithi130 ceph-mon[82829]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/3974784268"}]': finished 2024-04-22T21:44:42.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:41 smithi130 ceph-mon[82829]: osdmap e162: 8 total, 8 up, 8 in 2024-04-22T21:44:42.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:41 smithi130 ceph-mon[82829]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:44:42.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:41 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/3747709372' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3568156832"}]: dispatch 2024-04-22T21:44:42.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:41 smithi130 ceph-mon[82829]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3568156832"}]: dispatch 2024-04-22T21:44:42.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:41 smithi150 ceph-mon[65103]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:6801/3974784268"}]': finished 2024-04-22T21:44:42.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:41 smithi150 ceph-mon[65103]: osdmap e162: 8 total, 8 up, 8 in 2024-04-22T21:44:42.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:41 smithi150 ceph-mon[65103]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-22T21:44:42.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:41 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/3747709372' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3568156832"}]: dispatch 2024-04-22T21:44:42.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:41 smithi150 ceph-mon[65103]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3568156832"}]: dispatch 2024-04-22T21:44:43.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:42 smithi130 ceph-mon[82829]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3568156832"}]': finished 2024-04-22T21:44:43.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:42 smithi130 ceph-mon[82829]: osdmap e163: 8 total, 8 up, 8 in 2024-04-22T21:44:43.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:42 smithi130 ceph-mon[85205]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3568156832"}]': finished 2024-04-22T21:44:43.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:42 smithi130 ceph-mon[85205]: osdmap e163: 8 total, 8 up, 8 in 2024-04-22T21:44:43.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:42 smithi150 ceph-mon[65103]: from='client.? ' entity='client.iscsi.foo.smithi130.pcsiph' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.130:0/3568156832"}]': finished 2024-04-22T21:44:43.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:42 smithi150 ceph-mon[65103]: osdmap e163: 8 total, 8 up, 8 in 2024-04-22T21:44:43.810 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:44:44.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:43 smithi130 ceph-mon[82829]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s 2024-04-22T21:44:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:43 smithi130 ceph-mon[85205]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s 2024-04-22T21:44:44.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:43 smithi150 ceph-mon[65103]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 723 B/s rd, 0 op/s 2024-04-22T21:44:44.148 INFO:teuthology.orchestra.run.smithi130.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi130.pcsiph on host smithi130 failed." 2024-04-22T21:44:44.919 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps' 2024-04-22T21:44:45.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:44 smithi130 ceph-mon[82829]: from='client.64828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:45.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:44 smithi130 ceph-mon[82829]: from='client.64834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:45.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:44 smithi130 ceph-mon[85205]: from='client.64828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:45.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:44 smithi130 ceph-mon[85205]: from='client.64834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:45.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:44 smithi150 ceph-mon[65103]: from='client.64828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:45.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:44 smithi150 ceph-mon[65103]: from='client.64834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:45.838 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:45 smithi130 ceph-mon[82829]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:45.838 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:45 smithi130 ceph-mon[85205]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:46.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:45 smithi150 ceph-mon[65103]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:46.966 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:44:46.966 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (11m) 74s ago 33m 27.3M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:44:46.966 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (10m) 74s ago 33m 82.9M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:44:46.966 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (10m) 74s ago 32m 62.9M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:44:46.966 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (12m) 74s ago 38m 454M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:44:46.966 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (22m) 74s ago 39m 515M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:44:46.966 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (8m) 74s ago 39m 64.4M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (9m) 74s ago 38m 52.8M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (8m) 74s ago 38m 48.1M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (12m) 74s ago 34m 16.3M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (12m) 74s ago 34m 15.1M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (6m) 74s ago 37m 56.7M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (6m) 74s ago 37m 61.5M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (5m) 74s ago 37m 47.6M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 027e1ac8c177 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (4m) 74s ago 36m 59.0M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 ca6c16e44590 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (3m) 74s ago 36m 58.9M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 0c80f1b448cb 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (3m) 74s ago 35m 50.9M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 aef1942f3f6b 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (2m) 74s ago 35m 54.8M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 d84338a30cd0 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (2m) 74s ago 35m 55.0M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 d0d3f15f9e88 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (11m) 74s ago 34m 72.8M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:44:46.967 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (84s) 74s ago 32m 75.2M - 19.0.0-3198-g56e81df3 82086b19bb07 f0fe6481306a 2024-04-22T21:44:46.968 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (79s) 74s ago 32m 75.0M - 19.0.0-3198-g56e81df3 82086b19bb07 b226f7943446 2024-04-22T21:44:47.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:46 smithi130 ceph-mon[85205]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:47.044 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:46 smithi130 ceph-mon[82829]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:47.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:46 smithi150 ceph-mon[65103]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:47.607 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions' 2024-04-22T21:44:47.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:47 smithi130 ceph-mon[85205]: from='client.64840 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:47.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:47 smithi130 ceph-mon[85205]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 706 B/s rd, 0 op/s 2024-04-22T21:44:47.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:47 smithi130 ceph-mon[82829]: from='client.64840 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:47.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:47 smithi130 ceph-mon[82829]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 706 B/s rd, 0 op/s 2024-04-22T21:44:48.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:47 smithi150 ceph-mon[65103]: from='client.64840 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:44:48.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:47 smithi150 ceph-mon[65103]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 706 B/s rd, 0 op/s 2024-04-22T21:44:49.069 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:44:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:44:48] "GET /metrics HTTP/1.1" 200 38425 "" "Prometheus/2.43.0" 2024-04-22T21:44:49.596 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:44:49.596 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:44:49.596 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:44:49.596 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:44:49.596 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:44:49.596 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:44:49.596 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:44:49.596 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:44:49.596 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 8 2024-04-22T21:44:49.596 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:44:49.596 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:44:49.597 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:44:49.597 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:44:49.597 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:44:49.597 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 15 2024-04-22T21:44:49.597 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:44:49.597 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:44:50.205 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:50 smithi130 ceph-mon[82829]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:50.205 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:50 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/799010138' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:50.206 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:50 smithi130 ceph-mon[85205]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:50.206 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:50 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/799010138' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:50.231 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-22T21:44:50.347 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:50 smithi150 ceph-mon[65103]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:50.347 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:50 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/799010138' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:44:51.347 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:44:51.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:44:51.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:44:52.108 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:52 smithi130 ceph-mon[82829]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:44:52.108 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:52 smithi130 ceph-mon[85205]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:44:52.216 INFO:teuthology.orchestra.run.smithi130.stdout:wait for servicemap items w/ changing names to refresh 2024-04-22T21:44:52.347 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:52 smithi150 ceph-mon[65103]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T21:44:52.881 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'sleep 60' 2024-04-22T21:44:54.278 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:54 smithi130 ceph-mon[82829]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 911 B/s rd, 0 op/s 2024-04-22T21:44:54.278 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:54 smithi130 ceph-mon[85205]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 911 B/s rd, 0 op/s 2024-04-22T21:44:54.347 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:54 smithi150 ceph-mon[65103]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 911 B/s rd, 0 op/s 2024-04-22T21:44:56.294 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:56 smithi130 ceph-mon[82829]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:56.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:56 smithi130 ceph-mon[85205]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:56.347 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:56 smithi150 ceph-mon[65103]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:44:57.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:57 smithi150 ceph-mon[65103]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:57.543 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:57 smithi130 ceph-mon[82829]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:57.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:57 smithi130 ceph-mon[85205]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:44:58.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:44:58 smithi130 ceph-mon[82829]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:58.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:44:58 smithi130 ceph-mon[85205]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:58.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:44:58 smithi150 ceph-mon[65103]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:44:59.294 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:44:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:44:58] "GET /metrics HTTP/1.1" 200 38419 "" "Prometheus/2.43.0" 2024-04-22T21:45:00.543 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:00 smithi130 ceph-mon[82829]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:00.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:00 smithi130 ceph-mon[85205]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:00.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:00 smithi150 ceph-mon[65103]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:02.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:02 smithi130 ceph-mon[85205]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:02.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:02 smithi130 ceph-mon[82829]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:02.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:02 smithi150 ceph-mon[65103]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:04.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:04 smithi130 ceph-mon[85205]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:04.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:04 smithi130 ceph-mon[82829]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:04.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:04 smithi150 ceph-mon[65103]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:06.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:06 smithi130 ceph-mon[82829]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:06.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:45:06.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:06 smithi130 ceph-mon[85205]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:06.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:45:06.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:06 smithi150 ceph-mon[65103]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:06.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:45:07.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:07 smithi150 ceph-mon[65103]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:07.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:07 smithi130 ceph-mon[85205]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:07.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:07 smithi130 ceph-mon[82829]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:08.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:08 smithi150 ceph-mon[65103]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:08.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:08 smithi130 ceph-mon[82829]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:08.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:08 smithi130 ceph-mon[85205]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:09.293 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:45:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:45:08] "GET /metrics HTTP/1.1" 200 38418 "" "Prometheus/2.43.0" 2024-04-22T21:45:10.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:10 smithi150 ceph-mon[65103]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:10.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:10 smithi130 ceph-mon[82829]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:10.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:10 smithi130 ceph-mon[85205]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:12.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:12 smithi150 ceph-mon[65103]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:12.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:12 smithi130 ceph-mon[82829]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:12.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:12 smithi130 ceph-mon[85205]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:14.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:14 smithi150 ceph-mon[65103]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:14.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:14 smithi130 ceph-mon[82829]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:14.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:14 smithi130 ceph-mon[85205]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:16.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:16 smithi150 ceph-mon[65103]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:16.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:16 smithi130 ceph-mon[82829]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:16.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:16 smithi130 ceph-mon[85205]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:17.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:17 smithi150 ceph-mon[65103]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:17.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:17 smithi130 ceph-mon[82829]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:17.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:17 smithi130 ceph-mon[85205]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:18.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:18 smithi150 ceph-mon[65103]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:18.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:18 smithi130 ceph-mon[85205]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:18.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:18 smithi130 ceph-mon[82829]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:19.293 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:45:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:45:18] "GET /metrics HTTP/1.1" 200 38418 "" "Prometheus/2.43.0" 2024-04-22T21:45:20.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:20 smithi150 ceph-mon[65103]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:20.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:20 smithi130 ceph-mon[85205]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:20.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:20 smithi130 ceph-mon[82829]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:21.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:45:21.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:45:21.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:45:22.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:22 smithi150 ceph-mon[65103]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:22.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:22 smithi130 ceph-mon[85205]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:22.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:22 smithi130 ceph-mon[82829]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:24.596 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:24 smithi130 ceph-mon[82829]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:24.596 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:24 smithi130 ceph-mon[85205]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:24.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:24 smithi150 ceph-mon[65103]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:26.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:26 smithi150 ceph-mon[65103]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:26.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:26 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:45:26.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:26 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:45:26.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:26 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:45:26.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:26 smithi130 ceph-mon[85205]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:26.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:26 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:45:26.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:26 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:45:26.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:26 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:45:26.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:26 smithi130 ceph-mon[82829]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:26.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:26 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T21:45:26.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:26 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T21:45:26.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:26 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' 2024-04-22T21:45:27.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:27 smithi150 ceph-mon[65103]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:27.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:27 smithi130 ceph-mon[85205]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:27.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:27 smithi130 ceph-mon[82829]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:28.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:28 smithi150 ceph-mon[65103]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:28.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:28 smithi130 ceph-mon[82829]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:28.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:28 smithi130 ceph-mon[85205]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:29.299 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:45:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:45:28] "GET /metrics HTTP/1.1" 200 38420 "" "Prometheus/2.43.0" 2024-04-22T21:45:30.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:30 smithi150 ceph-mon[65103]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:30.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:30 smithi130 ceph-mon[85205]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:30.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:30 smithi130 ceph-mon[82829]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:32.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:32 smithi130 ceph-mon[85205]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:32.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:32 smithi130 ceph-mon[82829]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:32.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:32 smithi150 ceph-mon[65103]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:34.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:34 smithi130 ceph-mon[82829]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:34.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:34 smithi130 ceph-mon[85205]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:34.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:34 smithi150 ceph-mon[65103]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:36.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:36 smithi130 ceph-mon[85205]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:36.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:36 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:45:36.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:36 smithi130 ceph-mon[82829]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:36.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:36 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:45:36.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:36 smithi150 ceph-mon[65103]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:36.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:36 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:45:37.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:37 smithi130 ceph-mon[82829]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:37.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:37 smithi130 ceph-mon[85205]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:37.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:37 smithi150 ceph-mon[65103]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:38.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:38 smithi130 ceph-mon[82829]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:38.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:38 smithi130 ceph-mon[85205]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:38.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:38 smithi150 ceph-mon[65103]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:39.293 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:45:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:45:38] "GET /metrics HTTP/1.1" 200 38420 "" "Prometheus/2.43.0" 2024-04-22T21:45:40.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:40 smithi130 ceph-mon[82829]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:40.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:40 smithi130 ceph-mon[85205]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:40.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:40 smithi150 ceph-mon[65103]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:42.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:42 smithi130 ceph-mon[85205]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:42.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:42 smithi130 ceph-mon[82829]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:42.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:42 smithi150 ceph-mon[65103]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:44.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:44 smithi130 ceph-mon[85205]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:44.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:44 smithi130 ceph-mon[82829]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:44.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:44 smithi150 ceph-mon[65103]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:45:46.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:46 smithi130 ceph-mon[85205]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:46.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:46 smithi130 ceph-mon[82829]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:46.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:46 smithi150 ceph-mon[65103]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:47.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:47 smithi130 ceph-mon[85205]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:47.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:47 smithi130 ceph-mon[82829]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:47.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:47 smithi150 ceph-mon[65103]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:48.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:48 smithi130 ceph-mon[82829]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:48.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:48 smithi130 ceph-mon[85205]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:48.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:48 smithi150 ceph-mon[65103]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:49.293 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:45:48 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:45:48] "GET /metrics HTTP/1.1" 200 38420 "" "Prometheus/2.43.0" 2024-04-22T21:45:50.649 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:50 smithi150 ceph-mon[65103]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:50.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:50 smithi130 ceph-mon[82829]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:50.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:50 smithi130 ceph-mon[85205]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:51.670 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:51 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:45:51.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:51 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:45:51.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:51 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:45:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:52 smithi130 ceph-mon[85205]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:52.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:52 smithi130 ceph-mon[82829]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:52.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:52 smithi150 ceph-mon[65103]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:54.722 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:54 smithi130 ceph-mon[82829]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:54.722 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:54 smithi130 ceph-mon[85205]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:54.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:54 smithi150 ceph-mon[65103]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:55.485 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ps' 2024-04-22T21:45:56.792 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:56 smithi130 ceph-mon[82829]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:56.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:56 smithi130 ceph-mon[85205]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:56.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:56 smithi150 ceph-mon[65103]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:45:57.498 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T21:45:57.498 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (12m) 2m ago 34m 27.3M - 0.25.0 c8568f914cd2 49cc4394f696 2024-04-22T21:45:57.498 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi150 *:3000 running (11m) 2m ago 34m 82.9M - 9.4.12 2bacad6d85d8 5e5fa903c090 2024-04-22T21:45:57.498 INFO:teuthology.orchestra.run.smithi130.stdout:iscsi.foo.smithi130.pcsiph smithi130 running (11m) 2m ago 33m 62.9M - 3.5 e1d6a67b021e 992367db94cf 2024-04-22T21:45:57.498 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi150 *:8443,9283,8765 running (13m) 2m ago 39m 454M - 19.0.0-3198-g56e81df3 82086b19bb07 db4c3cf52264 2024-04-22T21:45:57.499 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (23m) 2m ago 40m 515M - 19.0.0-3198-g56e81df3 82086b19bb07 2854a3fbb395 2024-04-22T21:45:57.499 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (10m) 2m ago 41m 64.4M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 9db2f699adb7 2024-04-22T21:45:57.499 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi150 running (10m) 2m ago 39m 52.8M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 241734f80ee1 2024-04-22T21:45:57.499 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (9m) 2m ago 39m 48.1M 2048M 19.0.0-3198-g56e81df3 82086b19bb07 cfb56e816511 2024-04-22T21:45:57.499 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (13m) 2m ago 35m 16.3M - 1.5.0 0da6a335fe13 660779e0e835 2024-04-22T21:45:57.499 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi150 *:9100 running (13m) 2m ago 35m 15.1M - 1.5.0 0da6a335fe13 d5ffcbabfa8c 2024-04-22T21:45:57.499 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (7m) 2m ago 39m 56.7M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 de61cc1c93b8 2024-04-22T21:45:57.499 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (7m) 2m ago 38m 61.5M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 3a370ce64fec 2024-04-22T21:45:57.499 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (6m) 2m ago 38m 47.6M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 027e1ac8c177 2024-04-22T21:45:57.499 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (5m) 2m ago 37m 59.0M 2992M 19.0.0-3198-g56e81df3 82086b19bb07 ca6c16e44590 2024-04-22T21:45:57.499 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi150 running (4m) 2m ago 37m 58.9M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 0c80f1b448cb 2024-04-22T21:45:57.500 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi150 running (4m) 2m ago 37m 50.9M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 aef1942f3f6b 2024-04-22T21:45:57.500 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi150 running (3m) 2m ago 36m 54.8M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 d84338a30cd0 2024-04-22T21:45:57.500 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi150 running (3m) 2m ago 36m 55.0M 3248M 19.0.0-3198-g56e81df3 82086b19bb07 d0d3f15f9e88 2024-04-22T21:45:57.500 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi150 *:9095 running (12m) 2m ago 35m 72.8M - 2.43.0 a07b618ecd1d a8e5ee53ede3 2024-04-22T21:45:57.500 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi130.dfmola smithi130 *:8000 running (2m) 2m ago 33m 75.2M - 19.0.0-3198-g56e81df3 82086b19bb07 f0fe6481306a 2024-04-22T21:45:57.500 INFO:teuthology.orchestra.run.smithi130.stdout:rgw.foo.smithi150.xqvixd smithi150 *:8000 running (2m) 2m ago 33m 75.0M - 19.0.0-3198-g56e81df3 82086b19bb07 b226f7943446 2024-04-22T21:45:57.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:57 smithi130 ceph-mon[82829]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:57.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:57 smithi130 ceph-mon[85205]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:57.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:57 smithi150 ceph-mon[65103]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:45:58.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions' 2024-04-22T21:45:58.770 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:58 smithi130 ceph-mon[85205]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:58.771 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:45:58 smithi130 ceph-mon[85205]: from='client.64852 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:45:58.771 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:58 smithi130 ceph-mon[82829]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:58.771 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:45:58 smithi130 ceph-mon[82829]: from='client.64852 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:45:58.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:58 smithi150 ceph-mon[65103]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:45:58.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:45:58 smithi150 ceph-mon[65103]: from='client.64852 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:45:59.043 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:45:58 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:45:58] "GET /metrics HTTP/1.1" 200 38420 "" "Prometheus/2.43.0" 2024-04-22T21:46:00.315 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:46:00.315 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-04-22T21:46:00.315 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 3 2024-04-22T21:46:00.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:46:00.315 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-04-22T21:46:00.315 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:46:00.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:46:00.315 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-04-22T21:46:00.315 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 8 2024-04-22T21:46:00.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:46:00.316 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw": { 2024-04-22T21:46:00.316 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 2 2024-04-22T21:46:00.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-04-22T21:46:00.316 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-04-22T21:46:00.316 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)": 15 2024-04-22T21:46:00.316 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-04-22T21:46:00.316 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:46:00.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:00 smithi130 ceph-mon[85205]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:00.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:00 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/3484443129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:46:00.641 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:00 smithi130 ceph-mon[82829]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:00.641 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:00 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/3484443129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:46:00.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:00 smithi150 ceph-mon[65103]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:00.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:00 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/3484443129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:46:00.970 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch upgrade status' 2024-04-22T21:46:02.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:02 smithi150 ceph-mon[65103]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:46:02.599 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:02 smithi130 ceph-mon[82829]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:46:02.599 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:02 smithi130 ceph-mon[85205]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T21:46:03.067 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:46:03.067 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7", 2024-04-22T21:46:03.067 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-04-22T21:46:03.067 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T21:46:03.067 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-04-22T21:46:03.067 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr", 2024-04-22T21:46:03.067 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw", 2024-04-22T21:46:03.067 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-04-22T21:46:03.067 INFO:teuthology.orchestra.run.smithi130.stdout: "osd" 2024-04-22T21:46:03.067 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-04-22T21:46:03.067 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "15/21 daemons upgraded", 2024-04-22T21:46:03.068 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi130.pcsiph on host smithi130 failed.", 2024-04-22T21:46:03.068 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": true 2024-04-22T21:46:03.068 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:46:03.740 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph health detail' 2024-04-22T21:46:04.680 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:04 smithi130 ceph-mon[85205]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:04.680 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:04 smithi130 ceph-mon[85205]: from='client.64864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:04.680 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:04 smithi130 ceph-mon[82829]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:04.681 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:04 smithi130 ceph-mon[82829]: from='client.64864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:04.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:04 smithi150 ceph-mon[65103]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:04.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:04 smithi150 ceph-mon[65103]: from='client.64864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:05.720 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_WARN Upgrading daemon iscsi.foo.smithi130.pcsiph on host smithi130 failed. 2024-04-22T21:46:05.720 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi130.pcsiph on host smithi130 failed. 2024-04-22T21:46:05.720 INFO:teuthology.orchestra.run.smithi130.stdout: Upgrade daemon: iscsi.foo.smithi130.pcsiph: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi130.pcsiph ... 2024-04-22T21:46:05.720 INFO:teuthology.orchestra.run.smithi130.stdout:Creating ceph-iscsi config... 2024-04-22T21:46:05.720 INFO:teuthology.orchestra.run.smithi130.stdout:Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/iscsi-gateway.cfg 2024-04-22T21:46:05.720 INFO:teuthology.orchestra.run.smithi130.stdout:Write file: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/iscsi.foo.smithi130.pcsiph/tcmu-runner-entrypoint.sh 2024-04-22T21:46:05.721 INFO:teuthology.orchestra.run.smithi130.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2ddd7d46b8\x2d00eb\x2d11ef\x2dbc93\x2dc7b262605968.slice/ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service 2024-04-22T21:46:05.721 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 1 from systemctl start ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph 2024-04-22T21:46:05.721 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stderr Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:46:05.721 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stderr See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:46:05.721 INFO:teuthology.orchestra.run.smithi130.stdout:Traceback (most recent call last): 2024-04-22T21:46:05.721 INFO:teuthology.orchestra.run.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T21:46:05.721 INFO:teuthology.orchestra.run.smithi130.stdout: return _run_code(code, main_globals, None, 2024-04-22T21:46:05.721 INFO:teuthology.orchestra.run.smithi130.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T21:46:05.721 INFO:teuthology.orchestra.run.smithi130.stdout: exec(code, run_globals) 2024-04-22T21:46:05.721 INFO:teuthology.orchestra.run.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5618, in 2024-04-22T21:46:05.721 INFO:teuthology.orchestra.run.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 5606, in main 2024-04-22T21:46:05.721 INFO:teuthology.orchestra.run.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3096, in command_deploy_from 2024-04-22T21:46:05.721 INFO:teuthology.orchestra.run.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3131, in _common_deploy 2024-04-22T21:46:05.722 INFO:teuthology.orchestra.run.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 3151, in _deploy_daemon_container 2024-04-22T21:46:05.722 INFO:teuthology.orchestra.run.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1087, in deploy_daemon 2024-04-22T21:46:05.722 INFO:teuthology.orchestra.run.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/__main__.py", line 1223, in deploy_daemon_units 2024-04-22T21:46:05.722 INFO:teuthology.orchestra.run.smithi130.stdout: File "/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/cephadm.e31ca4716d00da92240374b06df95e06dd890b951fabc12934151562def20805/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T21:46:05.722 INFO:teuthology.orchestra.run.smithi130.stdout:RuntimeError: Failed command: systemctl start ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph: Job for ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service failed because the control process exited with error code. 2024-04-22T21:46:05.722 INFO:teuthology.orchestra.run.smithi130.stdout:See "systemctl status ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" and "journalctl -xeu ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@iscsi.foo.smithi130.pcsiph.service" for details. 2024-04-22T21:46:06.431 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-22T21:46:06.735 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:06 smithi130 ceph-mon[82829]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:06.735 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:06 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:46:06.736 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:06 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/1760492018' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:46:06.736 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:06 smithi130 ceph-mon[85205]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:06.736 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:06 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:46:06.736 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:06 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/1760492018' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:46:06.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:06 smithi150 ceph-mon[65103]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:06.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:06 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:46:06.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:06 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/1760492018' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T21:46:07.757 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:07 smithi130 ceph-mon[82829]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:46:07.757 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:07 smithi130 ceph-mon[85205]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:46:07.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:07 smithi150 ceph-mon[65103]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:46:08.538 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-04-22T21:46:08.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:08 smithi130 ceph-mon[85205]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:08.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:08 smithi130 ceph-mon[82829]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:08.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:08 smithi150 ceph-mon[65103]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:09.146 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:08 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:46:08] "GET /metrics HTTP/1.1" 200 38419 "" "Prometheus/2.43.0" 2024-04-22T21:46:09.170 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-22T21:46:09.707 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:09 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/2095792411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:46:09.707 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:09 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/2095792411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:46:09.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:09 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/2095792411' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:46:10.777 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:10 smithi130 ceph-mon[82829]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:10.777 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:10 smithi130 ceph-mon[85205]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:10.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:10 smithi150 ceph-mon[65103]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:11.293 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-3198-g56e81df3 (56e81df3aeb98b717efe2ab0537a8c60249f95f7) squid (dev)" 2024-04-22T21:46:11.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:11 smithi130 ceph-mon[82829]: from='client.? 172.21.15.130:0/1119988091' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:46:11.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:11 smithi130 ceph-mon[85205]: from='client.? 172.21.15.130:0/1119988091' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:46:11.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:11 smithi150 ceph-mon[65103]: from='client.? 172.21.15.130:0/1119988091' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T21:46:12.018 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -e sha1=56e81df3aeb98b717efe2ab0537a8c60249f95f7 -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-04-22T21:46:12.573 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:12 smithi130 ceph-mon[82829]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:12.573 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:12 smithi130 ceph-mon[85205]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:12.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:12 smithi150 ceph-mon[65103]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:14.221 INFO:teuthology.orchestra.run.smithi130.stdout:osd 8 2m ago - 2024-04-22T21:46:14.526 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:14 smithi130 ceph-mon[82829]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:14.526 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:14 smithi130 ceph-mon[85205]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:14.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:14 smithi150 ceph-mon[65103]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:14.876 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T21:46:14.886 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi130.front.sepia.ceph.com 2024-04-22T21:46:14.886 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-04-22T21:46:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:15 smithi130 ceph-mon[85205]: from='client.64888 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:15.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:15 smithi130 ceph-mon[82829]: from='client.64888 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:15.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:15 smithi150 ceph-mon[65103]: from='client.64888 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:16.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:16 smithi130 ceph-mon[82829]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:16.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:16 smithi130 ceph-mon[85205]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:16.849 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:16 smithi150 ceph-mon[65103]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:17.470 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-04-22T21:46:17.470 INFO:teuthology.orchestra.run.smithi130.stdout: "image": "quay.io/ceph/ceph", 2024-04-22T21:46:17.470 INFO:teuthology.orchestra.run.smithi130.stdout: "registry": "quay.io", 2024-04-22T21:46:17.470 INFO:teuthology.orchestra.run.smithi130.stdout: "bare_image": "ceph/ceph", 2024-04-22T21:46:17.470 INFO:teuthology.orchestra.run.smithi130.stdout: "versions": [] 2024-04-22T21:46:17.470 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-04-22T21:46:17.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:17 smithi130 ceph-mon[85205]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:46:17.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:17 smithi130 ceph-mon[82829]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:46:17.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:17 smithi150 ceph-mon[65103]: from='client.64777 -' entity='client.iscsi.foo.smithi130.pcsiph' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T21:46:18.202 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-04-22T21:46:18.779 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:18 smithi130 ceph-mon[85205]: from='client.64894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:18.779 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:18 smithi130 ceph-mon[85205]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:18.780 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:18 smithi130 ceph-mon[82829]: from='client.64894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:18.780 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:18 smithi130 ceph-mon[82829]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:18.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:18 smithi150 ceph-mon[65103]: from='client.64894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:18.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:18 smithi150 ceph-mon[65103]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:19.043 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:18 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:46:18] "GET /metrics HTTP/1.1" 200 38419 "" "Prometheus/2.43.0" 2024-04-22T21:46:20.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:20 smithi130 ceph-mon[85205]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:20.794 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:20 smithi130 ceph-mon[82829]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:20.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:20 smithi150 ceph-mon[65103]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T21:46:20.895 INFO:teuthology.orchestra.run.smithi130.stdout: "16.2.0", 2024-04-22T21:46:21.481 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-04-22T21:46:21.786 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:21 smithi130 ceph-mon[82829]: from='client.64900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:21.786 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:21 smithi130 ceph-mon[82829]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:46:21.786 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:21 smithi130 ceph-mon[85205]: from='client.64900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:21.787 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:21 smithi130 ceph-mon[85205]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:46:21.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:21 smithi150 ceph-mon[65103]: from='client.64900 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:21.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:21 smithi150 ceph-mon[65103]: from='mgr.54105 172.21.15.130:0/3326038657' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T21:46:22.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:22 smithi130 ceph-mon[85205]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:22.749 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:22 smithi130 ceph-mon[82829]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:22.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:22 smithi150 ceph-mon[65103]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: "v16.2.2", 2024-04-22T21:46:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: "v16.2.2-20210505", 2024-04-22T21:46:24.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:24 smithi130 ceph-mon[85205]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:24.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:24 smithi130 ceph-mon[85205]: from='client.64906 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:24.732 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:24 smithi130 ceph-mon[82829]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:24.732 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:24 smithi130 ceph-mon[82829]: from='client.64906 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:24.755 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-22T21:46:24.765 INFO:tasks.cephadm:Teardown begin 2024-04-22T21:46:24.766 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T21:46:24.792 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T21:46:24.820 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:24 smithi150 ceph-mon[65103]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T21:46:24.820 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:24 smithi150 ceph-mon[65103]: from='client.64906 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T21:46:24.825 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-22T21:46:24.826 DEBUG:teuthology.orchestra.run.smithi130:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-22T21:46:24.847 DEBUG:teuthology.orchestra.run.smithi150:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-22T21:46:24.883 INFO:tasks.cephadm:Stopping all daemons... 2024-04-22T21:46:24.883 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-22T21:46:24.883 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.a 2024-04-22T21:46:25.043 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:24 smithi130 systemd[1]: Stopping Ceph mon.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:46:25.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a[82825]: 2024-04-22T21:46:25.198+0000 7f3e144be700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:46:25.489 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a[82825]: 2024-04-22T21:46:25.198+0000 7f3e144be700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-22T21:46:25.489 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:25 smithi130 podman[126590]: 2024-04-22 21:46:25.219436474 +0000 UTC m=+0.122922920 container died 9db2f699adb7aae2a77ebd428c29aa03d2d7d53147a071e1a364d002397e76f8 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-22T21:46:25.489 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:46:25] ENGINE Bus STOPPING 2024-04-22T21:46:25.489 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:46:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T21:46:25.489 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:46:25] ENGINE Bus STOPPED 2024-04-22T21:46:25.489 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:46:25] ENGINE Bus STARTING 2024-04-22T21:46:25.489 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: CherryPy Checker: 2024-04-22T21:46:25.489 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: The Application mounted at '' has an empty config. 2024-04-22T21:46:25.489 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:46:25.489 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:46:25] ENGINE Serving on http://:::9283 2024-04-22T21:46:25.490 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:25 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: [22/Apr/2024:21:46:25] ENGINE Bus STARTED 2024-04-22T21:46:25.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:25 smithi130 podman[126590]: 2024-04-22 21:46:25.615471801 +0000 UTC m=+0.518958248 container cleanup 9db2f699adb7aae2a77ebd428c29aa03d2d7d53147a071e1a364d002397e76f8 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-56e81df, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-22T21:46:25.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Apr 22 21:46:25 smithi130 bash[126590]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-a 2024-04-22T21:46:26.014 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.a.service' 2024-04-22T21:46:26.060 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:46:26.060 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-22T21:46:26.060 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-22T21:46:26.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.c 2024-04-22T21:46:26.429 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:26 smithi130 systemd[1]: Stopping Ceph mon.c for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:46:26.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:26 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c[85201]: 2024-04-22T21:46:26.352+0000 7f3b599be700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:46:26.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:26 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c[85201]: 2024-04-22T21:46:26.352+0000 7f3b599be700 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-04-22T21:46:26.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:26 smithi130 podman[126723]: 2024-04-22 21:46:26.556736448 +0000 UTC m=+0.287733169 container died cfb56e816511afdcf5ef78d1abd8665fb06c5ea2a9264ef5fe1107a2a66750ba (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, RELEASE=main-56e81df, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-22T21:46:27.189 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:26 smithi130 podman[126723]: 2024-04-22 21:46:26.978715955 +0000 UTC m=+0.709712661 container cleanup cfb56e816511afdcf5ef78d1abd8665fb06c5ea2a9264ef5fe1107a2a66750ba (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2) 2024-04-22T21:46:27.189 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:26 smithi130 bash[126723]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c 2024-04-22T21:46:27.432 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.c.service' 2024-04-22T21:46:27.453 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:27 smithi130 podman[126745]: 2024-04-22 21:46:27.230429676 +0000 UTC m=+0.672410065 container remove cfb56e816511afdcf5ef78d1abd8665fb06c5ea2a9264ef5fe1107a2a66750ba (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-c, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS) 2024-04-22T21:46:27.453 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:27 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-22T21:46:27.454 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:27 smithi130 systemd[1]: Stopped Ceph mon.c for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:46:27.454 INFO:journalctl@ceph.mon.c.smithi130.stdout:Apr 22 21:46:27 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.c.service: Consumed 6.422s CPU time. 2024-04-22T21:46:27.469 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:46:27.469 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-22T21:46:27.469 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-22T21:46:27.469 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.b 2024-04-22T21:46:27.783 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:27 smithi150 systemd[1]: Stopping Ceph mon.b for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:46:28.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b[65099]: 2024-04-22T21:46:27.802+0000 7fc18e495700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:46:28.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b[65099]: 2024-04-22T21:46:27.802+0000 7fc18e495700 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-04-22T21:46:28.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:28 smithi150 podman[99117]: 2024-04-22 21:46:28.007432929 +0000 UTC m=+0.325686024 container died 241734f80ee17280373c5cf45aa3d3de3896576eb62a6308926551e512fdb00d (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-56e81df, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T21:46:28.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:28 smithi150 podman[99117]: 2024-04-22 21:46:28.487093528 +0000 UTC m=+0.805346622 container cleanup 241734f80ee17280373c5cf45aa3d3de3896576eb62a6308926551e512fdb00d (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-22T21:46:28.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Apr 22 21:46:28 smithi150 bash[99117]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mon-b 2024-04-22T21:46:28.838 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mon.b.service' 2024-04-22T21:46:28.886 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:46:28.887 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-22T21:46:28.887 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-22T21:46:28.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.y 2024-04-22T21:46:29.205 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:28 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: ::ffff:172.21.15.150 - - [22/Apr/2024:21:46:28] "GET /metrics HTTP/1.1" 200 38418 "" "Prometheus/2.43.0" 2024-04-22T21:46:29.205 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:28 smithi130 systemd[1]: Stopping Ceph mgr.y for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:46:29.505 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:46:29.224+0000 7fd75885e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:46:29.506 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:29 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y[60100]: 2024-04-22T21:46:29.224+0000 7fd75885e700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-22T21:46:29.506 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:29 smithi130 podman[126857]: 2024-04-22 21:46:29.256177295 +0000 UTC m=+0.162976536 container died 2854a3fbb395345090e04b7d6a0c5e98baceef935557417cf9744b695541bc12 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416) 2024-04-22T21:46:29.781 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:29 smithi130 podman[126857]: 2024-04-22 21:46:29.600618161 +0000 UTC m=+0.507417395 container cleanup 2854a3fbb395345090e04b7d6a0c5e98baceef935557417cf9744b695541bc12 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, GIT_CLEAN=True) 2024-04-22T21:46:29.781 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Apr 22 21:46:29 smithi130 bash[126857]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-y 2024-04-22T21:46:30.003 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.y.service' 2024-04-22T21:46:30.037 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:46:30.038 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-22T21:46:30.038 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-22T21:46:30.038 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.x 2024-04-22T21:46:30.347 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:46:30 smithi150 systemd[1]: Stopping Ceph mgr.x for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:46:30.630 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:46:30 smithi150 podman[99236]: 2024-04-22 21:46:30.389105356 +0000 UTC m=+0.138970750 container died db4c3cf522640658cd9cd07fe8fc26d758e3f288b94dc7a957d5a98675ed17b8 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-56e81df, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T21:46:30.981 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:46:30 smithi150 podman[99236]: 2024-04-22 21:46:30.825173467 +0000 UTC m=+0.575038858 container cleanup db4c3cf522640658cd9cd07fe8fc26d758e3f288b94dc7a957d5a98675ed17b8 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-04-22T21:46:30.981 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:46:30 smithi150 bash[99236]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x 2024-04-22T21:46:31.229 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.x.service' 2024-04-22T21:46:31.249 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:46:31 smithi150 podman[99250]: 2024-04-22 21:46:31.001768855 +0000 UTC m=+0.610158428 container remove db4c3cf522640658cd9cd07fe8fc26d758e3f288b94dc7a957d5a98675ed17b8 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-22T21:46:31.249 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:46:31 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-22T21:46:31.249 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:46:31 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-22T21:46:31.250 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:46:31 smithi150 systemd[1]: Stopped Ceph mgr.x for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:46:31.250 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Apr 22 21:46:31 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@mgr.x.service: Consumed 23.242s CPU time. 2024-04-22T21:46:31.262 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:46:31.262 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-22T21:46:31.262 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-22T21:46:31.262 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.0 2024-04-22T21:46:31.543 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:31 smithi130 systemd[1]: Stopping Ceph osd.0 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:46:32.043 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:31 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0[92998]: 2024-04-22T21:46:31.582+0000 7f6782815700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:46:32.043 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:31 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0[92998]: 2024-04-22T21:46:31.582+0000 7f6782815700 -1 osd.0 163 *** Got signal Terminated *** 2024-04-22T21:46:32.043 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:31 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0[92998]: 2024-04-22T21:46:31.582+0000 7f6782815700 -1 osd.0 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:46:33.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:46:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:46:33.552Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2024-04-22T21:46:33.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:46:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:46:33.552Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-22T21:46:37.293 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:37 smithi130 podman[126978]: 2024-04-22 21:46:37.031992735 +0000 UTC m=+5.549142281 container died de61cc1c93b805af234561fc5a63ce81c03762495326c97e110601c6511e82a3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-22T21:46:37.643 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:37 smithi130 podman[126978]: 2024-04-22 21:46:37.404983456 +0000 UTC m=+5.922133008 container cleanup de61cc1c93b805af234561fc5a63ce81c03762495326c97e110601c6511e82a3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=) 2024-04-22T21:46:37.643 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:37 smithi130 bash[126978]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0 2024-04-22T21:46:38.003 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:37 smithi130 podman[126996]: 2024-04-22 21:46:37.664348916 +0000 UTC m=+0.627560109 container remove de61cc1c93b805af234561fc5a63ce81c03762495326c97e110601c6511e82a3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416) 2024-04-22T21:46:38.256 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:38 smithi130 podman[127090]: 2024-04-22 21:46:37.92635151 +0000 UTC m=+0.018316360 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:46:38.256 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:38 smithi130 podman[127090]: 2024-04-22 21:46:38.040256445 +0000 UTC m=+0.132221272 container create a512fdd3e120fcbc76654d23bc46dbb0f53dfdaa01b4608f81775b595e8f72c4 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, RELEASE=main-56e81df, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5) 2024-04-22T21:46:38.543 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:38 smithi130 podman[127090]: 2024-04-22 21:46:38.276510197 +0000 UTC m=+0.368475058 container init a512fdd3e120fcbc76654d23bc46dbb0f53dfdaa01b4608f81775b595e8f72c4 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-56e81df, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-22T21:46:38.543 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:38 smithi130 podman[127090]: 2024-04-22 21:46:38.280591056 +0000 UTC m=+0.372555891 container start a512fdd3e120fcbc76654d23bc46dbb0f53dfdaa01b4608f81775b595e8f72c4 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-deactivate, RELEASE=main-56e81df, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:46:38.543 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:38 smithi130 podman[127090]: 2024-04-22 21:46:38.384616341 +0000 UTC m=+0.476581182 container attach a512fdd3e120fcbc76654d23bc46dbb0f53dfdaa01b4608f81775b595e8f72c4 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240416, RELEASE=main-56e81df, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-22T21:46:38.842 INFO:journalctl@ceph.osd.0.smithi130.stdout:Apr 22 21:46:38 smithi130 podman[127090]: 2024-04-22 21:46:38.566665366 +0000 UTC m=+0.658630202 container died a512fdd3e120fcbc76654d23bc46dbb0f53dfdaa01b4608f81775b595e8f72c4 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-22T21:46:39.131 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.0.service' 2024-04-22T21:46:39.171 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:46:39.172 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-22T21:46:39.172 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-22T21:46:39.172 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.1 2024-04-22T21:46:39.388 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:39 smithi130 systemd[1]: Stopping Ceph osd.1 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:46:39.793 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1[97944]: 2024-04-22T21:46:39.576+0000 7f3576b0b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:46:39.793 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1[97944]: 2024-04-22T21:46:39.576+0000 7f3576b0b700 -1 osd.1 163 *** Got signal Terminated *** 2024-04-22T21:46:39.793 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:39 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1[97944]: 2024-04-22T21:46:39.576+0000 7f3576b0b700 -1 osd.1 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:46:43.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:46:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:46:43.552Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi130:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-22T21:46:43.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:46:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:46:43.553Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2024-04-22T21:46:43.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:46:43 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:46:43.553Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-22T21:46:45.204 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:44 smithi130 podman[127195]: 2024-04-22 21:46:44.910075709 +0000 UTC m=+5.450931215 container died 3a370ce64feceff6250574d2622d7f19e0f73968ecf8e3415fd4854d58847147 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-56e81df, org.label-schema.schema-version=1.0) 2024-04-22T21:46:45.204 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:45 smithi130 podman[127195]: 2024-04-22 21:46:45.224890701 +0000 UTC m=+5.765746189 container cleanup 3a370ce64feceff6250574d2622d7f19e0f73968ecf8e3415fd4854d58847147 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=main-56e81df) 2024-04-22T21:46:45.543 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:45 smithi130 bash[127195]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1 2024-04-22T21:46:45.543 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:45 smithi130 podman[127221]: 2024-04-22 21:46:45.459664937 +0000 UTC m=+0.542533984 container remove 3a370ce64feceff6250574d2622d7f19e0f73968ecf8e3415fd4854d58847147 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T21:46:46.043 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:45 smithi130 podman[127316]: 2024-04-22 21:46:45.699117638 +0000 UTC m=+0.018150579 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:46:46.044 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:45 smithi130 podman[127316]: 2024-04-22 21:46:45.810622742 +0000 UTC m=+0.129655680 container create 0033a912c89c8fa19cf57132437cda49c21bd5fac9864f59c077d151d8dea3c3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-56e81df, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:46:46.044 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:46 smithi130 podman[127316]: 2024-04-22 21:46:46.029637366 +0000 UTC m=+0.348670304 container init 0033a912c89c8fa19cf57132437cda49c21bd5fac9864f59c077d151d8dea3c3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-deactivate, RELEASE=main-56e81df, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:46:46.044 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:46 smithi130 podman[127316]: 2024-04-22 21:46:46.033196554 +0000 UTC m=+0.352229500 container start 0033a912c89c8fa19cf57132437cda49c21bd5fac9864f59c077d151d8dea3c3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=main-56e81df, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-22T21:46:46.544 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:46 smithi130 podman[127316]: 2024-04-22 21:46:46.129910001 +0000 UTC m=+0.448942955 container attach 0033a912c89c8fa19cf57132437cda49c21bd5fac9864f59c077d151d8dea3c3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-22T21:46:46.544 INFO:journalctl@ceph.osd.1.smithi130.stdout:Apr 22 21:46:46 smithi130 podman[127316]: 2024-04-22 21:46:46.331285433 +0000 UTC m=+0.650318369 container died 0033a912c89c8fa19cf57132437cda49c21bd5fac9864f59c077d151d8dea3c3 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True) 2024-04-22T21:46:46.893 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.1.service' 2024-04-22T21:46:46.930 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:46:46.930 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-22T21:46:46.930 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-22T21:46:46.930 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.2 2024-04-22T21:46:47.257 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:47 smithi130 systemd[1]: Stopping Ceph osd.2 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:46:47.543 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:47 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2[104119]: 2024-04-22T21:46:47.277+0000 7f2c0fdda700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:46:47.543 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:47 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2[104119]: 2024-04-22T21:46:47.277+0000 7f2c0fdda700 -1 osd.2 163 *** Got signal Terminated *** 2024-04-22T21:46:47.543 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:47 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2[104119]: 2024-04-22T21:46:47.277+0000 7f2c0fdda700 -1 osd.2 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:46:52.983 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:52 smithi130 podman[127421]: 2024-04-22 21:46:52.697148711 +0000 UTC m=+5.524069216 container died 027e1ac8c1776484763e0d9835ceb8c47d3973c466603ec2edaefaa9c7cccabd (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2, ceph=True, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-56e81df, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:46:53.293 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:53 smithi130 podman[127421]: 2024-04-22 21:46:53.09432416 +0000 UTC m=+5.921244664 container cleanup 027e1ac8c1776484763e0d9835ceb8c47d3973c466603ec2edaefaa9c7cccabd (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-56e81df, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240416) 2024-04-22T21:46:53.293 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:53 smithi130 bash[127421]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2 2024-04-22T21:46:53.645 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:53 smithi130 podman[127439]: 2024-04-22 21:46:53.33736708 +0000 UTC m=+0.636545658 container remove 027e1ac8c1776484763e0d9835ceb8c47d3973c466603ec2edaefaa9c7cccabd (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-56e81df, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416) 2024-04-22T21:46:53.645 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:46:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:46:53.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi130:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2024-04-22T21:46:53.645 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:46:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:46:53.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2024-04-22T21:46:53.646 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:46:53 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:46:53.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-22T21:46:53.898 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:53 smithi130 podman[127530]: 2024-04-22 21:46:53.568726729 +0000 UTC m=+0.018187643 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:46:53.899 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:53 smithi130 podman[127530]: 2024-04-22 21:46:53.705043728 +0000 UTC m=+0.154504615 container create 4e88352f523cc9627c8fb9157b9953b3f3bae9125948cc5daaad4a30859089ff (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-22T21:46:53.899 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:53 smithi130 podman[127530]: 2024-04-22 21:46:53.915636105 +0000 UTC m=+0.365096999 container init 4e88352f523cc9627c8fb9157b9953b3f3bae9125948cc5daaad4a30859089ff (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df) 2024-04-22T21:46:53.899 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:53 smithi130 podman[127530]: 2024-04-22 21:46:53.919131612 +0000 UTC m=+0.368592501 container start 4e88352f523cc9627c8fb9157b9953b3f3bae9125948cc5daaad4a30859089ff (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True) 2024-04-22T21:46:54.193 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:54 smithi130 podman[127530]: 2024-04-22 21:46:54.015730312 +0000 UTC m=+0.465191203 container attach 4e88352f523cc9627c8fb9157b9953b3f3bae9125948cc5daaad4a30859089ff (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0) 2024-04-22T21:46:54.490 INFO:journalctl@ceph.osd.2.smithi130.stdout:Apr 22 21:46:54 smithi130 podman[127530]: 2024-04-22 21:46:54.214585284 +0000 UTC m=+0.664046191 container died 4e88352f523cc9627c8fb9157b9953b3f3bae9125948cc5daaad4a30859089ff (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-56e81df) 2024-04-22T21:46:54.866 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.2.service' 2024-04-22T21:46:54.903 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:46:54.903 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-22T21:46:54.903 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-22T21:46:54.903 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.3 2024-04-22T21:46:55.235 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:46:55 smithi130 systemd[1]: Stopping Ceph osd.3 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:46:55.543 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:46:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3[110318]: 2024-04-22T21:46:55.255+0000 7f5a06573700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:46:55.543 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:46:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3[110318]: 2024-04-22T21:46:55.255+0000 7f5a06573700 -1 osd.3 163 *** Got signal Terminated *** 2024-04-22T21:46:55.543 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:46:55 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3[110318]: 2024-04-22T21:46:55.255+0000 7f5a06573700 -1 osd.3 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:46:57.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:46:57 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:46:57.556+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:46:58.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:46:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:46:58.015+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:46:58.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:46:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:46:58.513+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:46:58.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:46:58 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:46:58.539+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:46:59.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:46:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:46:59.002+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:46:59.851 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:46:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:46:59.508+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:46:59.851 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:46:59 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:46:59.480+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:00.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:00 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:00.014+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:00.347 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:00 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:47:00.225+0000 7ff3ccea8700 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.064620+0000 front 2024-04-22T21:46:34.064777+0000 (oldest deadline 2024-04-22T21:46:59.364817+0000) 2024-04-22T21:47:00.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:00 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:00.545+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:00.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:00 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:00.495+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:00.861 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:47:00 smithi130 podman[127645]: 2024-04-22 21:47:00.617754716 +0000 UTC m=+5.463543849 container died ca6c16e44590fd32921fad575a182be35ab58b297cd5a96677559e9f3736f3ab (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, RELEASE=main-56e81df, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T21:47:01.210 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:47:00 smithi130 podman[127645]: 2024-04-22 21:47:00.971987934 +0000 UTC m=+5.817777061 container cleanup ca6c16e44590fd32921fad575a182be35ab58b297cd5a96677559e9f3736f3ab (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-56e81df) 2024-04-22T21:47:01.210 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:47:00 smithi130 bash[127645]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3 2024-04-22T21:47:01.347 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:01 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:47:01.206+0000 7ff3ccea8700 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.064620+0000 front 2024-04-22T21:46:34.064777+0000 (oldest deadline 2024-04-22T21:46:59.364817+0000) 2024-04-22T21:47:01.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:01 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:01.012+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:01.511 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:47:01 smithi130 podman[127661]: 2024-04-22 21:47:01.231422079 +0000 UTC m=+0.614683743 container remove ca6c16e44590fd32921fad575a182be35ab58b297cd5a96677559e9f3736f3ab (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-56e81df, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-22T21:47:01.793 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:47:01 smithi130 podman[127757]: 2024-04-22 21:47:01.532411105 +0000 UTC m=+0.101266322 container create e5bad96aeabc7b0447389095b180a55587df20cb054d08937e1d334ac21dc5fe (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-56e81df, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=) 2024-04-22T21:47:01.793 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:47:01 smithi130 podman[127757]: 2024-04-22 21:47:01.449516555 +0000 UTC m=+0.018371735 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:47:01.793 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:47:01 smithi130 podman[127757]: 2024-04-22 21:47:01.734971073 +0000 UTC m=+0.303826245 container init e5bad96aeabc7b0447389095b180a55587df20cb054d08937e1d334ac21dc5fe (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-56e81df, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-22T21:47:01.794 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:47:01 smithi130 podman[127757]: 2024-04-22 21:47:01.738021235 +0000 UTC m=+0.306876409 container start e5bad96aeabc7b0447389095b180a55587df20cb054d08937e1d334ac21dc5fe (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, RELEASE=main-56e81df, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-22T21:47:01.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:01 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:01.524+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:01.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:01 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:01.566+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:02.292 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:47:01 smithi130 podman[127757]: 2024-04-22 21:47:01.818253061 +0000 UTC m=+0.387108249 container attach e5bad96aeabc7b0447389095b180a55587df20cb054d08937e1d334ac21dc5fe (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, RELEASE=main-56e81df, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-22T21:47:02.293 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:47:02 smithi130 podman[127757]: 2024-04-22 21:47:02.02874966 +0000 UTC m=+0.597604837 container died e5bad96aeabc7b0447389095b180a55587df20cb054d08937e1d334ac21dc5fe (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=main-56e81df, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS) 2024-04-22T21:47:02.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:02.043+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:02.347 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:47:02.181+0000 7ff3ccea8700 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.064620+0000 front 2024-04-22T21:46:34.064777+0000 (oldest deadline 2024-04-22T21:46:59.364817+0000) 2024-04-22T21:47:02.543 INFO:journalctl@ceph.osd.3.smithi130.stdout:Apr 22 21:47:02 smithi130 podman[127783]: 2024-04-22 21:47:02.554806913 +0000 UTC m=+0.519510450 container remove e5bad96aeabc7b0447389095b180a55587df20cb054d08937e1d334ac21dc5fe (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-56e81df, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS) 2024-04-22T21:47:02.552 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.3.service' 2024-04-22T21:47:02.590 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:47:02.590 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-22T21:47:02.590 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-22T21:47:02.590 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.4 2024-04-22T21:47:02.847 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:02 smithi150 systemd[1]: Stopping Ceph osd.4 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:47:02.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:02.520+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:02.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:02.530+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:03.170 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:03.006+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:03.170 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:47:02.935+0000 7ff3d48bc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:47:03.171 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:47:02.935+0000 7ff3d48bc700 -1 osd.4 163 *** Got signal Terminated *** 2024-04-22T21:47:03.171 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:02 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:47:02.935+0000 7ff3d48bc700 -1 osd.4 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:47:03.514 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:47:03.190+0000 7ff3ccea8700 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.064620+0000 front 2024-04-22T21:46:34.064777+0000 (oldest deadline 2024-04-22T21:46:59.364817+0000) 2024-04-22T21:47:03.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:47:03.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi130:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2024-04-22T21:47:03.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:47:03.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2024-04-22T21:47:03.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:03 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:47:03.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-22T21:47:03.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:03.534+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:03.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:03 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:03.561+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:04.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:04.052+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:04.347 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:47:04.237+0000 7ff3ccea8700 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.064620+0000 front 2024-04-22T21:46:34.064777+0000 (oldest deadline 2024-04-22T21:46:59.364817+0000) 2024-04-22T21:47:04.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:04.586+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:04.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:04 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:04.525+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:05.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:05.082+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:05.347 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:47:05.201+0000 7ff3ccea8700 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.064620+0000 front 2024-04-22T21:46:34.064777+0000 (oldest deadline 2024-04-22T21:46:59.364817+0000) 2024-04-22T21:47:05.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:05.600+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:05.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:05 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:05.525+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:06.347 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:47:06.238+0000 7ff3ccea8700 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.064620+0000 front 2024-04-22T21:46:34.064777+0000 (oldest deadline 2024-04-22T21:46:59.364817+0000) 2024-04-22T21:47:06.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:06.094+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:06.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:06.563+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:06.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:06.524+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:07.347 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:07 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4[75721]: 2024-04-22T21:47:07.258+0000 7ff3ccea8700 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.064620+0000 front 2024-04-22T21:46:34.064777+0000 (oldest deadline 2024-04-22T21:46:59.364817+0000) 2024-04-22T21:47:07.348 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:07 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:07.092+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:07.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:07 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:07.557+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:07.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:07 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:07.547+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:08.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:08.054+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:08.684 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:08 smithi150 podman[99359]: 2024-04-22 21:47:08.40715988 +0000 UTC m=+5.602805113 container died 0c80f1b448cbfa6c485d32c41895f89f4e7c83d81c4d98a6ceaa22dd3f81abf4 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=main-56e81df, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=) 2024-04-22T21:47:08.684 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:08.515+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:08.684 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:08 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:08.557+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:09.025 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:08 smithi150 podman[99359]: 2024-04-22 21:47:08.878657281 +0000 UTC m=+6.074302513 container cleanup 0c80f1b448cbfa6c485d32c41895f89f4e7c83d81c4d98a6ceaa22dd3f81abf4 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4, RELEASE=main-56e81df, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416) 2024-04-22T21:47:09.026 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:08 smithi150 bash[99359]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4 2024-04-22T21:47:09.347 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:09 smithi150 podman[99376]: 2024-04-22 21:47:09.163665953 +0000 UTC m=+0.747057309 container remove 0c80f1b448cbfa6c485d32c41895f89f4e7c83d81c4d98a6ceaa22dd3f81abf4 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True) 2024-04-22T21:47:09.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:09.044+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:09.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:09.044+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:09.738 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:09.526+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:09.738 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:09.526+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.165832+0000 front 2024-04-22T21:46:44.165906+0000 (oldest deadline 2024-04-22T21:47:08.865830+0000) 2024-04-22T21:47:09.738 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:09.522+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:09.738 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:09 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:09.522+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:09.739 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:09 smithi150 podman[99472]: 2024-04-22 21:47:09.407710025 +0000 UTC m=+0.018454448 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:47:09.739 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:09 smithi150 podman[99472]: 2024-04-22 21:47:09.531501083 +0000 UTC m=+0.142245531 container create 45cc38f89b5b23c1ad76c7ef2922f32f1bd70aa2f7ac19b3a7596a6dbe77603b (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-56e81df, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:47:10.036 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:09 smithi150 podman[99472]: 2024-04-22 21:47:09.758974806 +0000 UTC m=+0.369719212 container init 45cc38f89b5b23c1ad76c7ef2922f32f1bd70aa2f7ac19b3a7596a6dbe77603b (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-56e81df, org.label-schema.build-date=20240416) 2024-04-22T21:47:10.036 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:09 smithi150 podman[99472]: 2024-04-22 21:47:09.762171069 +0000 UTC m=+0.372915475 container start 45cc38f89b5b23c1ad76c7ef2922f32f1bd70aa2f7ac19b3a7596a6dbe77603b (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0) 2024-04-22T21:47:10.036 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:09 smithi150 podman[99472]: 2024-04-22 21:47:09.842246597 +0000 UTC m=+0.452991004 container attach 45cc38f89b5b23c1ad76c7ef2922f32f1bd70aa2f7ac19b3a7596a6dbe77603b (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-56e81df, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416) 2024-04-22T21:47:10.325 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:10 smithi150 podman[99472]: 2024-04-22 21:47:10.056818784 +0000 UTC m=+0.667563196 container died 45cc38f89b5b23c1ad76c7ef2922f32f1bd70aa2f7ac19b3a7596a6dbe77603b (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=main-56e81df, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-22T21:47:10.325 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:10.077+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:10.325 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:10.077+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:10.577 INFO:journalctl@ceph.osd.4.smithi150.stdout:Apr 22 21:47:10 smithi150 podman[99498]: 2024-04-22 21:47:10.595663207 +0000 UTC m=+0.529857353 container remove 45cc38f89b5b23c1ad76c7ef2922f32f1bd70aa2f7ac19b3a7596a6dbe77603b (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-56e81df, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-22T21:47:10.578 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:10.471+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:10.578 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:10.471+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:10.578 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:10.480+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:10.578 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:10.480+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.165832+0000 front 2024-04-22T21:46:44.165906+0000 (oldest deadline 2024-04-22T21:47:08.865830+0000) 2024-04-22T21:47:10.594 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.4.service' 2024-04-22T21:47:10.630 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:47:10.630 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-22T21:47:10.631 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-22T21:47:10.631 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.5 2024-04-22T21:47:10.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:10 smithi150 systemd[1]: Stopping Ceph osd.5 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:47:11.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:11.063+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:11.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:11.063+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:11.347 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:10.989+0000 7ff9695c5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:47:11.347 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:10.989+0000 7ff9695c5700 -1 osd.5 163 *** Got signal Terminated *** 2024-04-22T21:47:11.347 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:10 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:10.989+0000 7ff9695c5700 -1 osd.5 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:47:11.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:11.466+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:11.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:11.466+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:11.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:11.496+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:11.848 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:11.496+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.165832+0000 front 2024-04-22T21:46:44.165906+0000 (oldest deadline 2024-04-22T21:47:08.865830+0000) 2024-04-22T21:47:11.848 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:11.593Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.130:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.130:8765: connect: connection refused" 2024-04-22T21:47:11.848 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:11.595Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.130:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.130:8765: connect: connection refused" 2024-04-22T21:47:11.848 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:11.595Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.130:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.130:8765: connect: connection refused" 2024-04-22T21:47:11.848 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:11.598Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.130:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.130:8765: connect: connection refused" 2024-04-22T21:47:11.848 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:11 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:11.599Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.130:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.130:8765: connect: connection refused" 2024-04-22T21:47:12.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:12.074+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:12.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:12.074+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:12.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:12.456+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:12.857 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:12.456+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:12.858 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:12.487+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:12.858 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:12 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:12.487+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.165832+0000 front 2024-04-22T21:46:44.165906+0000 (oldest deadline 2024-04-22T21:47:08.865830+0000) 2024-04-22T21:47:13.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:13.103+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:13.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:13.103+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:13.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:47:13.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi130:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2024-04-22T21:47:13.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:47:13.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2024-04-22T21:47:13.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:13 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:47:13.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-22T21:47:13.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:13.485+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:13.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:13.485+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:13.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:13.518+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:13.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:13 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:13.518+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.165832+0000 front 2024-04-22T21:46:44.165906+0000 (oldest deadline 2024-04-22T21:47:08.865830+0000) 2024-04-22T21:47:14.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:14 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:14.061+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:14.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:14 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:14.061+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:14.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:14 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:14.061+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:49.496692+0000 front 2024-04-22T21:46:49.496705+0000 (oldest deadline 2024-04-22T21:47:13.596633+0000) 2024-04-22T21:47:14.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:14 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:14.452+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:14.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:14 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:14.452+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:14.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:14 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:14.481+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:14.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:14 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:14.481+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.165832+0000 front 2024-04-22T21:46:44.165906+0000 (oldest deadline 2024-04-22T21:47:08.865830+0000) 2024-04-22T21:47:14.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:14 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:14.481+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:48.866517+0000 front 2024-04-22T21:46:48.866443+0000 (oldest deadline 2024-04-22T21:47:14.166377+0000) 2024-04-22T21:47:15.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:15.098+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:15.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:15.098+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:15.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:15.098+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:49.496692+0000 front 2024-04-22T21:46:49.496705+0000 (oldest deadline 2024-04-22T21:47:13.596633+0000) 2024-04-22T21:47:15.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:15.485+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:15.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:15.485+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:15.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:15.468+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:34.964068+0000 front 2024-04-22T21:46:34.964144+0000 (oldest deadline 2024-04-22T21:46:57.264080+0000) 2024-04-22T21:47:15.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:15.468+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.165832+0000 front 2024-04-22T21:46:44.165906+0000 (oldest deadline 2024-04-22T21:47:08.865830+0000) 2024-04-22T21:47:15.847 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:15 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5[80265]: 2024-04-22T21:47:15.468+0000 7ff965bb9700 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:48.866517+0000 front 2024-04-22T21:46:48.866443+0000 (oldest deadline 2024-04-22T21:47:14.166377+0000) 2024-04-22T21:47:16.424 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:16.134+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:16.425 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:16.134+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:16.425 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:16.134+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:49.496692+0000 front 2024-04-22T21:46:49.496705+0000 (oldest deadline 2024-04-22T21:47:13.596633+0000) 2024-04-22T21:47:16.695 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:16 smithi150 podman[99578]: 2024-04-22 21:47:16.444809844 +0000 UTC m=+5.572470612 container died aef1942f3f6becfefcd9237db20907ca1e3e80b208a9828af0ddc01f41d526eb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=) 2024-04-22T21:47:16.695 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:16.478+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:16.696 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:16 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:16.478+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:17.095 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:16 smithi150 podman[99578]: 2024-04-22 21:47:16.90717988 +0000 UTC m=+6.034840648 container cleanup aef1942f3f6becfefcd9237db20907ca1e3e80b208a9828af0ddc01f41d526eb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-22T21:47:17.095 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:16 smithi150 bash[99578]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5 2024-04-22T21:47:17.347 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:17 smithi150 podman[99594]: 2024-04-22 21:47:17.175359857 +0000 UTC m=+0.731969886 container remove aef1942f3f6becfefcd9237db20907ca1e3e80b208a9828af0ddc01f41d526eb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=main-56e81df, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-22T21:47:17.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:17 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:17.115+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:17.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:17 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:17.115+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:17.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:17 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:17.115+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:49.496692+0000 front 2024-04-22T21:46:49.496705+0000 (oldest deadline 2024-04-22T21:47:13.596633+0000) 2024-04-22T21:47:17.749 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:17 smithi150 podman[99690]: 2024-04-22 21:47:17.437611826 +0000 UTC m=+0.018654803 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:47:17.749 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:17 smithi150 podman[99690]: 2024-04-22 21:47:17.551225601 +0000 UTC m=+0.132268565 container create 7df5daa85b984a3aba03161e00cd3e25d74e5d01e3c92d495976c066879dbbfb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-56e81df, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-22T21:47:17.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:17 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:17.436+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:17.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:17 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:17.436+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:17.750 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:17 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:17.436+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:18.038 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:17 smithi150 podman[99690]: 2024-04-22 21:47:17.770286662 +0000 UTC m=+0.351329626 container init 7df5daa85b984a3aba03161e00cd3e25d74e5d01e3c92d495976c066879dbbfb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-22T21:47:18.038 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:17 smithi150 podman[99690]: 2024-04-22 21:47:17.773358282 +0000 UTC m=+0.354401247 container start 7df5daa85b984a3aba03161e00cd3e25d74e5d01e3c92d495976c066879dbbfb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=main-56e81df, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-22T21:47:18.038 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:17 smithi150 podman[99690]: 2024-04-22 21:47:17.845379612 +0000 UTC m=+0.426422589 container attach 7df5daa85b984a3aba03161e00cd3e25d74e5d01e3c92d495976c066879dbbfb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-deactivate, RELEASE=main-56e81df, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True) 2024-04-22T21:47:18.327 INFO:journalctl@ceph.osd.5.smithi150.stdout:Apr 22 21:47:18 smithi150 podman[99690]: 2024-04-22 21:47:18.058849081 +0000 UTC m=+0.639892054 container died 7df5daa85b984a3aba03161e00cd3e25d74e5d01e3c92d495976c066879dbbfb (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=main-56e81df) 2024-04-22T21:47:18.328 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:18.161+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:18.328 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:18.161+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:18.328 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:18.161+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:49.496692+0000 front 2024-04-22T21:46:49.496705+0000 (oldest deadline 2024-04-22T21:47:13.596633+0000) 2024-04-22T21:47:18.597 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:18.442+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:18.597 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:18.442+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:18.597 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:18 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:18.443+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:18.729 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.5.service' 2024-04-22T21:47:18.767 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:47:18.767 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-22T21:47:18.768 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-22T21:47:18.768 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.6 2024-04-22T21:47:19.082 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:18 smithi150 systemd[1]: Stopping Ceph osd.6 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:47:19.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:19.102+0000 7f32283e0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:47:19.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:19.102+0000 7f32283e0700 -1 osd.6 163 *** Got signal Terminated *** 2024-04-22T21:47:19.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:19.102+0000 7f32283e0700 -1 osd.6 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:47:19.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:19.186+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:19.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:19.186+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:19.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:19.186+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:49.496692+0000 front 2024-04-22T21:46:49.496705+0000 (oldest deadline 2024-04-22T21:47:13.596633+0000) 2024-04-22T21:47:19.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:19.468+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:19.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:19.468+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:19.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:19 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:19.468+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:20.479 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:20.201+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:20.479 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:20.201+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:20.480 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:20.201+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:49.496692+0000 front 2024-04-22T21:46:49.496705+0000 (oldest deadline 2024-04-22T21:47:13.596633+0000) 2024-04-22T21:47:20.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:20.500+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:20.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:20.500+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:20.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:20 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:20.500+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:21.499 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:21 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:21.215+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:21.499 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:21 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:21.215+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:21.499 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:21 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:21.215+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:49.496692+0000 front 2024-04-22T21:46:49.496705+0000 (oldest deadline 2024-04-22T21:47:13.596633+0000) 2024-04-22T21:47:21.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:21 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:21.519+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:21.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:21 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:21.519+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:21.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:21 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:21.519+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:22.519 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:22.221+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:22.519 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:22.221+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:22.520 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:22.221+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:49.496692+0000 front 2024-04-22T21:46:49.496705+0000 (oldest deadline 2024-04-22T21:47:13.596633+0000) 2024-04-22T21:47:22.520 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:22.221+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-04-22T21:46:59.998045+0000 front 2024-04-22T21:46:59.998014+0000 (oldest deadline 2024-04-22T21:47:21.697898+0000) 2024-04-22T21:47:22.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:22.540+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:22.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:22.540+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:22.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:22 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:22.540+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:23.495 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:23.262+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:31.994040+0000 front 2024-04-22T21:46:31.994106+0000 (oldest deadline 2024-04-22T21:46:57.894194+0000) 2024-04-22T21:47:23.495 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:23.262+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:43.695777+0000 front 2024-04-22T21:46:43.695531+0000 (oldest deadline 2024-04-22T21:47:08.995633+0000) 2024-04-22T21:47:23.496 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:23.262+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:49.496692+0000 front 2024-04-22T21:46:49.496705+0000 (oldest deadline 2024-04-22T21:47:13.596633+0000) 2024-04-22T21:47:23.496 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6[85262]: 2024-04-22T21:47:23.262+0000 7f32209cc700 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-04-22T21:46:59.998045+0000 front 2024-04-22T21:46:59.998014+0000 (oldest deadline 2024-04-22T21:47:21.697898+0000) 2024-04-22T21:47:23.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:23 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:47:23.554Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi130:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2024-04-22T21:47:23.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:23 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:47:23.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2024-04-22T21:47:23.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:23 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:47:23.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-22T21:47:23.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:23.515+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:23.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:23.515+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:23.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:23 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:23.515+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:24.847 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:24 smithi150 podman[99795]: 2024-04-22 21:47:24.592593604 +0000 UTC m=+5.587640713 container died d84338a30cd03ad8437d550f2ac746f8da68674a020a0f21d4e01e06570ba613 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-56e81df, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD) 2024-04-22T21:47:24.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:24.479+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:24.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:24.479+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:24.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:24.479+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:24.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:24 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:24.479+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-04-22T21:46:58.597152+0000 front 2024-04-22T21:46:58.597137+0000 (oldest deadline 2024-04-22T21:47:23.897079+0000) 2024-04-22T21:47:25.224 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:24 smithi150 podman[99795]: 2024-04-22 21:47:24.952092156 +0000 UTC m=+5.947139267 container cleanup d84338a30cd03ad8437d550f2ac746f8da68674a020a0f21d4e01e06570ba613 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-56e81df, org.label-schema.schema-version=1.0) 2024-04-22T21:47:25.224 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:24 smithi150 bash[99795]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6 2024-04-22T21:47:25.560 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:25 smithi150 podman[99814]: 2024-04-22 21:47:25.244874458 +0000 UTC m=+0.649093364 container remove d84338a30cd03ad8437d550f2ac746f8da68674a020a0f21d4e01e06570ba613 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True) 2024-04-22T21:47:25.560 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:25 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:25.482+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:25.560 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:25 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:25.482+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:25.560 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:25 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:25.482+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:25.560 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:25 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:25.482+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-04-22T21:46:58.597152+0000 front 2024-04-22T21:46:58.597137+0000 (oldest deadline 2024-04-22T21:47:23.897079+0000) 2024-04-22T21:47:25.847 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:25 smithi150 podman[99906]: 2024-04-22 21:47:25.483936073 +0000 UTC m=+0.018187015 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:47:25.847 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:25 smithi150 podman[99906]: 2024-04-22 21:47:25.596294678 +0000 UTC m=+0.130545646 container create 8604bdf8958c4c18fcd503345ce10a263840ba191d774f95d026f6acdcc55738 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-56e81df, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-22T21:47:25.847 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:25 smithi150 podman[99906]: 2024-04-22 21:47:25.815170407 +0000 UTC m=+0.349421333 container init 8604bdf8958c4c18fcd503345ce10a263840ba191d774f95d026f6acdcc55738 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, ceph=True, RELEASE=main-56e81df, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True) 2024-04-22T21:47:25.847 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:25 smithi150 podman[99906]: 2024-04-22 21:47:25.818446797 +0000 UTC m=+0.352697722 container start 8604bdf8958c4c18fcd503345ce10a263840ba191d774f95d026f6acdcc55738 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-56e81df, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2) 2024-04-22T21:47:26.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:25 smithi150 podman[99906]: 2024-04-22 21:47:25.903688779 +0000 UTC m=+0.437939705 container attach 8604bdf8958c4c18fcd503345ce10a263840ba191d774f95d026f6acdcc55738 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD) 2024-04-22T21:47:26.347 INFO:journalctl@ceph.osd.6.smithi150.stdout:Apr 22 21:47:26 smithi150 podman[99906]: 2024-04-22 21:47:26.11760535 +0000 UTC m=+0.651856276 container died 8604bdf8958c4c18fcd503345ce10a263840ba191d774f95d026f6acdcc55738 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True) 2024-04-22T21:47:26.656 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:26.512+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:26.656 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:26.512+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:26.656 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:26.512+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:26.656 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:26 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:26.512+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-04-22T21:46:58.597152+0000 front 2024-04-22T21:46:58.597137+0000 (oldest deadline 2024-04-22T21:47:23.897079+0000) 2024-04-22T21:47:26.670 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.6.service' 2024-04-22T21:47:26.709 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:47:26.710 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-22T21:47:26.710 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-22T21:47:26.710 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.7 2024-04-22T21:47:27.043 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:26 smithi150 systemd[1]: Stopping Ceph osd.7 for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:47:27.043 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:27.063+0000 7f077d255700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T21:47:27.043 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:27.063+0000 7f077d255700 -1 osd.7 163 *** Got signal Terminated *** 2024-04-22T21:47:27.043 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:27.063+0000 7f077d255700 -1 osd.7 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T21:47:27.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:27.475+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:27.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:27.475+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:27.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:27.475+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:27.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:27 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:27.475+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-04-22T21:46:58.597152+0000 front 2024-04-22T21:46:58.597137+0000 (oldest deadline 2024-04-22T21:47:23.897079+0000) 2024-04-22T21:47:28.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:28 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:28.440+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:28.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:28 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:28.440+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:28.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:28 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:28.440+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:28.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:28 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:28.440+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-04-22T21:46:58.597152+0000 front 2024-04-22T21:46:58.597137+0000 (oldest deadline 2024-04-22T21:47:23.897079+0000) 2024-04-22T21:47:29.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:29.445+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:29.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:29.445+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:29.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:29.445+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:29.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:29.445+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-04-22T21:46:58.597152+0000 front 2024-04-22T21:46:58.597137+0000 (oldest deadline 2024-04-22T21:47:23.897079+0000) 2024-04-22T21:47:29.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:29 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:29.445+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.150:6804 osd.4 since back 2024-04-22T21:47:03.897595+0000 front 2024-04-22T21:47:03.897485+0000 (oldest deadline 2024-04-22T21:47:29.197462+0000) 2024-04-22T21:47:30.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:30 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:30.445+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:30.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:30 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:30.445+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:30.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:30 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:30.445+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:30.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:30 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:30.445+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-04-22T21:46:58.597152+0000 front 2024-04-22T21:46:58.597137+0000 (oldest deadline 2024-04-22T21:47:23.897079+0000) 2024-04-22T21:47:30.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:30 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:30.445+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.150:6804 osd.4 since back 2024-04-22T21:47:03.897595+0000 front 2024-04-22T21:47:03.897485+0000 (oldest deadline 2024-04-22T21:47:29.197462+0000) 2024-04-22T21:47:31.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:31 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:31.493+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-04-22T21:46:35.393292+0000 front 2024-04-22T21:46:35.393536+0000 (oldest deadline 2024-04-22T21:46:58.293489+0000) 2024-04-22T21:47:31.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:31 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:31.493+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-04-22T21:46:44.094962+0000 front 2024-04-22T21:46:44.095091+0000 (oldest deadline 2024-04-22T21:47:09.394809+0000) 2024-04-22T21:47:31.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:31 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:31.493+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-04-22T21:46:50.996201+0000 front 2024-04-22T21:46:50.996131+0000 (oldest deadline 2024-04-22T21:47:16.896025+0000) 2024-04-22T21:47:31.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:31 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:31.493+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-04-22T21:46:58.597152+0000 front 2024-04-22T21:46:58.597137+0000 (oldest deadline 2024-04-22T21:47:23.897079+0000) 2024-04-22T21:47:31.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:31 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7[89961]: 2024-04-22T21:47:31.493+0000 7f0775841700 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.150:6804 osd.4 since back 2024-04-22T21:47:03.897595+0000 front 2024-04-22T21:47:03.897485+0000 (oldest deadline 2024-04-22T21:47:29.197462+0000) 2024-04-22T21:47:32.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:32 smithi150 podman[100013]: 2024-04-22 21:47:32.602045839 +0000 UTC m=+5.647884601 container died d0d3f15f9e88f54c809e71fc5c81880c28a7b2dedef219960434273e18fbe55c (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-56e81df, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=) 2024-04-22T21:47:33.226 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:32 smithi150 podman[100013]: 2024-04-22 21:47:32.963082947 +0000 UTC m=+6.008921710 container cleanup d0d3f15f9e88f54c809e71fc5c81880c28a7b2dedef219960434273e18fbe55c (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=main-56e81df, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0) 2024-04-22T21:47:33.226 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:32 smithi150 bash[100013]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7 2024-04-22T21:47:33.563 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:33 smithi150 podman[100042]: 2024-04-22 21:47:33.247437916 +0000 UTC m=+0.646297910 container remove d0d3f15f9e88f54c809e71fc5c81880c28a7b2dedef219960434273e18fbe55c (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416) 2024-04-22T21:47:33.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:47:33.554Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=8 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi130:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-22T21:47:33.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:47:33.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi130:8443/api/prometheus_receiver\": dial tcp 172.21.15.130:8443: connect: connection refused" 2024-04-22T21:47:33.793 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:33 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:47:33.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-22T21:47:33.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:33 smithi150 podman[100135]: 2024-04-22 21:47:33.487620354 +0000 UTC m=+0.018140964 image pull 82086b19bb070a73a22a9572b4d46af73014eba1e8e605aa289115cdc1fd1de4 quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7 2024-04-22T21:47:33.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:33 smithi150 podman[100135]: 2024-04-22 21:47:33.61609166 +0000 UTC m=+0.146612270 container create 72475cc9d1af2c04a31fd5b6a38693809304065c3b28c9401ed9f03ff6f40590 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-56e81df, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T21:47:33.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:33 smithi150 podman[100135]: 2024-04-22 21:47:33.817576283 +0000 UTC m=+0.348096900 container init 72475cc9d1af2c04a31fd5b6a38693809304065c3b28c9401ed9f03ff6f40590 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-22T21:47:33.847 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:33 smithi150 podman[100135]: 2024-04-22 21:47:33.820992169 +0000 UTC m=+0.351512787 container start 72475cc9d1af2c04a31fd5b6a38693809304065c3b28c9401ed9f03ff6f40590 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-56e81df, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-22T21:47:34.347 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:33 smithi150 podman[100135]: 2024-04-22 21:47:33.892271636 +0000 UTC m=+0.422792262 container attach 72475cc9d1af2c04a31fd5b6a38693809304065c3b28c9401ed9f03ff6f40590 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-56e81df, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T21:47:34.347 INFO:journalctl@ceph.osd.7.smithi150.stdout:Apr 22 21:47:34 smithi150 podman[100135]: 2024-04-22 21:47:34.112817614 +0000 UTC m=+0.643338231 container died 72475cc9d1af2c04a31fd5b6a38693809304065c3b28c9401ed9f03ff6f40590 (image=quay.ceph.io/ceph-ci/ceph:56e81df3aeb98b717efe2ab0537a8c60249f95f7, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=main-56e81df, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240416) 2024-04-22T21:47:34.706 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@osd.7.service' 2024-04-22T21:47:34.744 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:47:34.745 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-22T21:47:34.745 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-22T21:47:34.745 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a 2024-04-22T21:47:35.080 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:34 smithi150 systemd[1]: Stopping Ceph prometheus.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:47:35.347 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:35.100Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T21:47:35.347 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:35.100Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-22T21:47:35.347 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:35.100Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-22T21:47:35.347 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:35.100Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T21:47:35.347 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:35.100Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-22T21:47:35.347 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:35.100Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-22T21:47:35.347 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:35.100Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T21:47:35.348 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:35.100Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-22T21:47:35.348 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:35.101Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-22T21:47:35.348 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:35.101Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T21:47:35.348 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:35.101Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-22T21:47:35.348 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a[61245]: ts=2024-04-22T21:47:35.101Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-22T21:47:35.348 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 podman[100240]: 2024-04-22 21:47:35.11092356 +0000 UTC m=+0.126120144 container died a8e5ee53ede38be89c93cd3198b62c6b1a0510fe4ee43ca432dbe4279cf3825e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:47:35.737 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 podman[100240]: 2024-04-22 21:47:35.540635227 +0000 UTC m=+0.555831853 container cleanup a8e5ee53ede38be89c93cd3198b62c6b1a0510fe4ee43ca432dbe4279cf3825e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:47:35.737 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 bash[100240]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T21:47:35.990 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Apr 22 21:47:35 smithi150 podman[100254]: 2024-04-22 21:47:35.758418896 +0000 UTC m=+0.649503309 container remove a8e5ee53ede38be89c93cd3198b62c6b1a0510fe4ee43ca432dbe4279cf3825e (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T21:47:35.994 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@prometheus.a.service' 2024-04-22T21:47:36.027 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T21:47:36.027 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-22T21:47:36.028 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-22T21:47:37.793 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:47:37 smithi130 systemd[1]: Stopping Ceph node-exporter.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:47:37.793 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:47:37 smithi130 podman[128147]: 2024-04-22 21:47:37.7733585 +0000 UTC m=+0.102171405 container died 660779e0e8355413972dd02c02df7233cfd48d046629a652a97664be53b2d833 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:47:38.293 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:47:38 smithi130 podman[128147]: 2024-04-22 21:47:38.087920436 +0000 UTC m=+0.416733353 container cleanup 660779e0e8355413972dd02c02df7233cfd48d046629a652a97664be53b2d833 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:47:38.293 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:47:38 smithi130 bash[128147]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a 2024-04-22T21:47:38.546 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:47:38 smithi130 podman[128161]: 2024-04-22 21:47:38.314043215 +0000 UTC m=+0.540948257 container remove 660779e0e8355413972dd02c02df7233cfd48d046629a652a97664be53b2d833 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T21:47:38.546 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:47:38 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-22T21:47:38.546 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:47:38 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-22T21:47:38.546 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:47:38 smithi130 systemd[1]: Stopped Ceph node-exporter.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:47:38.546 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Apr 22 21:47:38 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 3.133s CPU time. 2024-04-22T21:47:38.962 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:38 smithi130 systemd[1]: Stopping Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:47:39.280 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:38 smithi130 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a[76462]: ts=2024-04-22T21:47:38.983Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-22T21:47:39.280 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:39 smithi130 podman[128268]: 2024-04-22 21:47:39.024558808 +0000 UTC m=+0.151386956 container died 49cc4394f696e3d77ce5d78fc7595ef45a90984cca68a514642d5c9bc55298a0 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:47:39.543 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:39 smithi130 podman[128268]: 2024-04-22 21:47:39.400573746 +0000 UTC m=+0.527401893 container cleanup 49cc4394f696e3d77ce5d78fc7595ef45a90984cca68a514642d5c9bc55298a0 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:47:39.543 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:39 smithi130 bash[128268]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a 2024-04-22T21:47:39.935 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:39 smithi130 podman[128282]: 2024-04-22 21:47:39.685170439 +0000 UTC m=+0.663656391 container remove 49cc4394f696e3d77ce5d78fc7595ef45a90984cca68a514642d5c9bc55298a0 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T21:47:39.936 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:39 smithi130 podman[128282]: 2024-04-22 21:47:39.752388391 +0000 UTC m=+0.730874297 volume remove cc9e6614baf79bae72709d72911f42c6120ac9f2b632e32b186cf68b8413bf2c 2024-04-22T21:47:40.293 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:39 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-22T21:47:40.293 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:39 smithi130 systemd[1]: Stopped Ceph alertmanager.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:47:40.293 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Apr 22 21:47:39 smithi130 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.694s CPU time. 2024-04-22T21:48:03.034 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-22T21:48:04.808 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:48:04 smithi150 systemd[1]: Stopping Ceph node-exporter.b for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:48:05.097 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:48:04 smithi150 podman[100649]: 2024-04-22 21:48:04.845550249 +0000 UTC m=+0.124900904 container died d5ffcbabfa8ced7ba8c567c2eabc482d8100dc08148f3a0c71f4f85403b34858 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:48:05.457 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:48:05 smithi150 podman[100649]: 2024-04-22 21:48:05.285170686 +0000 UTC m=+0.564521399 container cleanup d5ffcbabfa8ced7ba8c567c2eabc482d8100dc08148f3a0c71f4f85403b34858 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:48:05.457 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:48:05 smithi150 bash[100649]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b 2024-04-22T21:48:05.847 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:48:05 smithi150 podman[100664]: 2024-04-22 21:48:05.478046802 +0000 UTC m=+0.634671162 container remove d5ffcbabfa8ced7ba8c567c2eabc482d8100dc08148f3a0c71f4f85403b34858 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T21:48:05.847 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:48:05 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-22T21:48:05.847 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:48:05 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-22T21:48:05.847 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:48:05 smithi150 systemd[1]: Stopped Ceph node-exporter.b for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:48:05.847 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Apr 22 21:48:05 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 3.162s CPU time. 2024-04-22T21:48:06.344 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:48:05 smithi150 systemd[1]: Stopping Ceph grafana.a for dd7d46b8-00eb-11ef-bc93-c7b262605968... 2024-04-22T21:48:06.344 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:48:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=server t=2024-04-22T21:48:06.114728679Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-04-22T21:48:06.344 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:48:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=tracing t=2024-04-22T21:48:06.115091396Z level=info msg="Closing tracing" 2024-04-22T21:48:06.344 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:48:06 smithi150 ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a[63484]: logger=ticker t=2024-04-22T21:48:06.115108556Z level=info msg=stopped last_tick=2024-04-22T21:48:00Z 2024-04-22T21:48:06.344 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:48:06 smithi150 podman[100770]: 2024-04-22 21:48:06.124983215 +0000 UTC m=+0.104039535 container died 5e5fa903c0902d89394cadb2f6cdf4e77f9237736819f85d8d0c9d7ce45e5387 (image=quay.io/ceph/grafana:9.4.12, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-22T21:48:06.596 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:48:06 smithi150 podman[100770]: 2024-04-22 21:48:06.439294077 +0000 UTC m=+0.418350405 container cleanup 5e5fa903c0902d89394cadb2f6cdf4e77f9237736819f85d8d0c9d7ce45e5387 (image=quay.io/ceph/grafana:9.4.12, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-22T21:48:06.597 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:48:06 smithi150 bash[100770]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a 2024-04-22T21:48:06.917 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:48:06 smithi150 podman[100784]: 2024-04-22 21:48:06.640362557 +0000 UTC m=+0.513823892 container remove 5e5fa903c0902d89394cadb2f6cdf4e77f9237736819f85d8d0c9d7ce45e5387 (image=quay.io/ceph/grafana:9.4.12, name=ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-22T21:48:06.918 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:48:06 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-22T21:48:06.918 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:48:06 smithi150 systemd[1]: Stopped Ceph grafana.a for dd7d46b8-00eb-11ef-bc93-c7b262605968. 2024-04-22T21:48:06.918 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Apr 22 21:48:06 smithi150 systemd[1]: ceph-dd7d46b8-00eb-11ef-bc93-c7b262605968@grafana.a.service: Consumed 11.863s CPU time. 2024-04-22T21:48:18.449 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T21:48:18.482 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T21:48:18.510 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-22T21:48:18.512 DEBUG:teuthology.misc:Transferring archived files from smithi130:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-22_20:08:13-orch-main-distro-default-smithi/7668628/remote/smithi130/crash 2024-04-22T21:48:18.513 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/crash -- . 2024-04-22T21:48:18.559 INFO:teuthology.orchestra.run.smithi130.stderr:tar: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-22T21:48:18.559 INFO:teuthology.orchestra.run.smithi130.stderr:tar: Error is not recoverable: exiting now 2024-04-22T21:48:18.561 DEBUG:teuthology.misc:Transferring archived files from smithi150:/var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-22_20:08:13-orch-main-distro-default-smithi/7668628/remote/smithi150/crash 2024-04-22T21:48:18.562 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar c -f - -C /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/crash -- . 2024-04-22T21:48:18.605 INFO:teuthology.orchestra.run.smithi150.stderr:tar: /var/lib/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-22T21:48:18.605 INFO:teuthology.orchestra.run.smithi150.stderr:tar: Error is not recoverable: exiting now 2024-04-22T21:48:18.606 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-22T21:48:18.607 DEBUG:teuthology.orchestra.run.smithi130:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-22T21:48:18.635 INFO:tasks.cephadm:Compressing logs... 2024-04-22T21:48:18.636 DEBUG:teuthology.orchestra.run.smithi130:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-22T21:48:18.678 DEBUG:teuthology.orchestra.run.smithi150:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-22T21:48:18.706 INFO:teuthology.orchestra.run.smithi130.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi130.dfmola.log 2024-04-22T21:48:18.706 INFO:teuthology.orchestra.run.smithi130.stderr:: No such file or directory 2024-04-22T21:48:18.706 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.log 2024-04-22T21:48:18.706 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-22T21:48:18.706 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi130.dfmola.log: /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-22T21:48:18.706 INFO:teuthology.orchestra.run.smithi130.stderr: 76.5% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi130.dfmola.log.gz 2024-04-22T21:48:18.706 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-22T21:48:18.707 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-22T21:48:18.707 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mgr.y.log 2024-04-22T21:48:18.707 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-22T21:48:18.707 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mon.c.log 2024-04-22T21:48:18.707 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mon.a.log 2024-04-22T21:48:18.709 INFO:teuthology.orchestra.run.smithi150.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi150.xqvixd.log 2024-04-22T21:48:18.709 INFO:teuthology.orchestra.run.smithi150.stderr:: No such file or directory 2024-04-22T21:48:18.709 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.log 2024-04-22T21:48:18.709 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi150.xqvixd.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-22T21:48:18.709 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-22T21:48:18.709 INFO:teuthology.orchestra.run.smithi150.stderr: 76.9% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi150.xqvixd.log.gz 2024-04-22T21:48:18.710 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-22T21:48:18.710 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-22T21:48:18.710 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mgr.x.log 2024-04-22T21:48:18.710 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-22T21:48:18.710 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-22T21:48:18.710 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-22T21:48:18.710 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.cephadm.log: 89.2% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-22T21:48:18.710 INFO:teuthology.orchestra.run.smithi150.stderr: 92.6% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-22T21:48:18.710 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mon.b.log 2024-04-22T21:48:18.710 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-22T21:48:18.710 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mon.b.log: 91.1% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-22T21:48:18.713 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mon.c.log: 94.0% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-22T21:48:18.714 INFO:teuthology.orchestra.run.smithi130.stderr: 94.4% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-22T21:48:18.719 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-22T21:48:18.720 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/tcmu-runner.log 2024-04-22T21:48:18.721 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cephadm.log: 91.0% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-22T21:48:18.723 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.cephadm.log: 95.4% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-22T21:48:18.732 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-22T21:48:18.733 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/tcmu-runner.log: 90.4% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/tcmu-runner.log.gz 2024-04-22T21:48:18.743 INFO:teuthology.orchestra.run.smithi150.stderr: 90.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-22T21:48:18.745 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-22T21:48:18.760 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 90.9% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-22T21:48:18.776 INFO:teuthology.orchestra.run.smithi130.stderr: 91.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-22T21:48:18.856 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mgr.x.log.gz 2024-04-22T21:48:19.225 INFO:teuthology.orchestra.run.smithi130.stderr: 89.1% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mgr.y.log.gz 2024-04-22T21:48:19.261 INFO:teuthology.orchestra.run.smithi150.stderr: 92.5% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mon.b.log.gz 2024-04-22T21:48:19.283 INFO:teuthology.orchestra.run.smithi130.stderr: 92.4% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mon.c.log.gz 2024-04-22T21:48:20.328 INFO:teuthology.orchestra.run.smithi130.stderr: 91.1% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-mon.a.log.gz 2024-04-22T21:48:20.373 INFO:teuthology.orchestra.run.smithi150.stderr: 93.2% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-22T21:48:20.460 INFO:teuthology.orchestra.run.smithi130.stderr: 93.3% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-22T21:48:20.580 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-22T21:48:20.761 INFO:teuthology.orchestra.run.smithi130.stderr: 93.4% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-22T21:48:20.761 INFO:teuthology.orchestra.run.smithi150.stderr: 93.6% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-22T21:48:20.769 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-22T21:48:20.771 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-04-22T21:48:20.771 INFO:teuthology.orchestra.run.smithi150.stderr:real 0m2.079s 2024-04-22T21:48:20.771 INFO:teuthology.orchestra.run.smithi150.stderr:user 0m8.113s 2024-04-22T21:48:20.771 INFO:teuthology.orchestra.run.smithi150.stderr:sys 0m0.342s 2024-04-22T21:48:21.093 INFO:teuthology.orchestra.run.smithi130.stderr: 93.4% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-22T21:48:21.100 INFO:teuthology.orchestra.run.smithi130.stderr: 93.4% -- replaced with /var/log/ceph/dd7d46b8-00eb-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-22T21:48:21.102 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-04-22T21:48:21.102 INFO:teuthology.orchestra.run.smithi130.stderr:real 0m2.411s 2024-04-22T21:48:21.102 INFO:teuthology.orchestra.run.smithi130.stderr:user 0m10.988s 2024-04-22T21:48:21.102 INFO:teuthology.orchestra.run.smithi130.stderr:sys 0m0.384s 2024-04-22T21:48:21.102 INFO:tasks.cephadm:Archiving logs... 2024-04-22T21:48:21.103 DEBUG:teuthology.misc:Transferring archived files from smithi130:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-22_20:08:13-orch-main-distro-default-smithi/7668628/remote/smithi130/log 2024-04-22T21:48:21.104 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-22T21:48:22.582 DEBUG:teuthology.misc:Transferring archived files from smithi150:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-22_20:08:13-orch-main-distro-default-smithi/7668628/remote/smithi150/log 2024-04-22T21:48:22.583 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-22T21:48:23.768 INFO:tasks.cephadm:Removing cluster... 2024-04-22T21:48:23.768 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 --force 2024-04-22T21:48:24.057 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid dd7d46b8-00eb-11ef-bc93-c7b262605968 --force 2024-04-22T21:48:24.349 INFO:tasks.cephadm:Removing cephadm ... 2024-04-22T21:48:24.349 DEBUG:teuthology.orchestra.run.smithi130:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-22T21:48:24.365 DEBUG:teuthology.orchestra.run.smithi150:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-22T21:48:24.380 INFO:tasks.cephadm:Teardown complete 2024-04-22T21:48:24.380 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-22T21:48:24.391 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-22T21:48:24.391 DEBUG:teuthology.orchestra.run.smithi130:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T21:48:24.408 DEBUG:teuthology.orchestra.run.smithi150:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T21:48:24.419 INFO:teuthology.orchestra.run.smithi130.stderr:bash: line 1: ntpq: command not found 2024-04-22T21:48:24.425 INFO:teuthology.orchestra.run.smithi130.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T21:48:24.425 INFO:teuthology.orchestra.run.smithi130.stdout:=============================================================================== 2024-04-22T21:48:24.425 INFO:teuthology.orchestra.run.smithi130.stdout:^- hv01.front.sepia.ceph.com 2 6 377 62 +1585us[+1582us] +/- 591ms 2024-04-22T21:48:24.425 INFO:teuthology.orchestra.run.smithi130.stdout:^* hv02.front.sepia.ceph.com 3 6 377 58 -309us[ -312us] +/- 57ms 2024-04-22T21:48:24.425 INFO:teuthology.orchestra.run.smithi130.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 61 +389us[ +386us] +/- 67ms 2024-04-22T21:48:24.425 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T21:48:24.433 INFO:teuthology.orchestra.run.smithi150.stderr:bash: line 1: ntpq: command not found 2024-04-22T21:48:24.438 INFO:teuthology.orchestra.run.smithi150.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T21:48:24.438 INFO:teuthology.orchestra.run.smithi150.stdout:=============================================================================== 2024-04-22T21:48:24.438 INFO:teuthology.orchestra.run.smithi150.stdout:^- hv01.front.sepia.ceph.com 2 6 377 57 +1541us[+1541us] +/- 591ms 2024-04-22T21:48:24.439 INFO:teuthology.orchestra.run.smithi150.stdout:^* hv02.front.sepia.ceph.com 3 6 377 64 -336us[ -336us] +/- 57ms 2024-04-22T21:48:24.439 INFO:teuthology.orchestra.run.smithi150.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 62 +354us[ +354us] +/- 67ms 2024-04-22T21:48:24.439 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T21:48:24.439 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-22T21:48:24.448 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-22T21:48:24.449 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-22T21:48:24.458 DEBUG:teuthology.orchestra.run.smithi130:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-22T21:48:24.552 DEBUG:teuthology.orchestra.run.smithi150:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-22T21:48:24.627 DEBUG:teuthology.orchestra.run.smithi130:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-22T21:48:24.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T21:48:24.655 DEBUG:teuthology.orchestra.run.smithi150:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-22T21:48:24.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T21:48:24.682 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-22T21:48:24.692 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-22T21:48:24.701 INFO:teuthology.task.internal:Duration was 2864.292566 seconds 2024-04-22T21:48:24.702 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-22T21:48:24.711 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-22T21:48:24.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-22T21:48:24.714 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-22T21:48:24.748 INFO:teuthology.orchestra.run.smithi130.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T21:48:24.758 INFO:teuthology.orchestra.run.smithi150.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T21:48:25.136 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-22T21:48:25.136 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi130.front.sepia.ceph.com 2024-04-22T21:48:25.136 DEBUG:teuthology.orchestra.run.smithi130:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-22T21:48:25.156 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi150.front.sepia.ceph.com 2024-04-22T21:48:25.156 DEBUG:teuthology.orchestra.run.smithi150:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-22T21:48:25.205 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-22T21:48:25.205 DEBUG:teuthology.orchestra.run.smithi130:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-22T21:48:25.208 DEBUG:teuthology.orchestra.run.smithi150:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-22T21:48:25.273 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-22T21:48:25.274 DEBUG:teuthology.orchestra.run.smithi130:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-22T21:48:25.276 DEBUG:teuthology.orchestra.run.smithi150:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-22T21:48:25.879 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-22T21:48:25.890 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-22T21:48:25.891 DEBUG:teuthology.orchestra.run.smithi130:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-22T21:48:25.914 DEBUG:teuthology.orchestra.run.smithi150:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-22T21:48:25.938 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-22T21:48:25.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-22T21:48:25.957 DEBUG:teuthology.orchestra.run.smithi150:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-22T21:48:25.976 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern = core 2024-04-22T21:48:26.001 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern = core 2024-04-22T21:48:26.027 DEBUG:teuthology.orchestra.run.smithi130:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-22T21:48:26.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T21:48:26.057 DEBUG:teuthology.orchestra.run.smithi150:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-22T21:48:26.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T21:48:26.082 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-22T21:48:26.092 INFO:teuthology.task.internal:Transferring archived files... 2024-04-22T21:48:26.093 DEBUG:teuthology.misc:Transferring archived files from smithi130:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-22_20:08:13-orch-main-distro-default-smithi/7668628/remote/smithi130 2024-04-22T21:48:26.093 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-22T21:48:26.147 DEBUG:teuthology.misc:Transferring archived files from smithi150:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-22_20:08:13-orch-main-distro-default-smithi/7668628/remote/smithi150 2024-04-22T21:48:26.148 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-22T21:48:26.186 INFO:teuthology.task.internal:Removing archive directory... 2024-04-22T21:48:26.186 DEBUG:teuthology.orchestra.run.smithi130:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-22T21:48:26.189 DEBUG:teuthology.orchestra.run.smithi150:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-22T21:48:26.233 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-22T21:48:26.245 INFO:teuthology.task.internal:Not uploading archives. 2024-04-22T21:48:26.245 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-22T21:48:26.254 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-22T21:48:26.255 DEBUG:teuthology.orchestra.run.smithi130:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-22T21:48:26.257 DEBUG:teuthology.orchestra.run.smithi150:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-22T21:48:26.269 INFO:teuthology.orchestra.run.smithi130.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 22 21:48 /home/ubuntu/cephtest 2024-04-22T21:48:26.288 INFO:teuthology.orchestra.run.smithi150.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 22 21:48 /home/ubuntu/cephtest 2024-04-22T21:48:26.289 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-22T21:48:26.300 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-22T21:48:26.321 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/connectivity} duration: 2864.2925662994385 owner: scheduled_teuthology@teuthology success: true 2024-04-22T21:48:26.321 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-22T21:48:26.404 INFO:teuthology.run:pass